2024-11-10 15:16:03,155 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 15:16:03,168 main DEBUG Took 0.010408 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 15:16:03,168 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 15:16:03,168 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 15:16:03,169 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 15:16:03,170 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 15:16:03,178 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 15:16:03,193 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 15:16:03,195 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 15:16:03,195 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 15:16:03,196 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 15:16:03,197 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 15:16:03,197 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 15:16:03,198 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 15:16:03,199 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 15:16:03,200 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 15:16:03,200 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 15:16:03,201 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 15:16:03,201 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 15:16:03,202 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 15:16:03,202 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 15:16:03,203 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 15:16:03,203 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 15:16:03,204 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 15:16:03,205 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 15:16:03,205 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 15:16:03,206 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 15:16:03,206 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 15:16:03,206 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 15:16:03,207 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 15:16:03,207 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 15:16:03,208 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 15:16:03,208 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 15:16:03,210 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 15:16:03,212 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 15:16:03,214 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 15:16:03,215 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 15:16:03,216 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 15:16:03,217 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 15:16:03,226 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 15:16:03,229 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 15:16:03,230 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 15:16:03,231 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 15:16:03,231 main DEBUG createAppenders(={Console}) 2024-11-10 15:16:03,232 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-10 15:16:03,233 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 15:16:03,233 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-10 15:16:03,234 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 15:16:03,234 main DEBUG OutputStream closed 2024-11-10 15:16:03,234 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 15:16:03,235 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 15:16:03,235 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-10 15:16:03,318 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 15:16:03,320 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 15:16:03,322 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 15:16:03,323 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 15:16:03,324 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 15:16:03,325 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 15:16:03,325 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 15:16:03,325 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 15:16:03,326 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 15:16:03,326 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 15:16:03,327 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 15:16:03,327 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 15:16:03,328 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 15:16:03,328 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 15:16:03,328 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 15:16:03,329 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 15:16:03,329 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 15:16:03,330 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 15:16:03,332 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 15:16:03,333 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-10 15:16:03,333 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 15:16:03,334 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-10T15:16:03,748 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d 2024-11-10 15:16:03,754 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 15:16:03,754 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T15:16:03,766 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-10T15:16:03,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T15:16:03,822 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/cluster_33e785d3-c89b-58a2-b49e-d3558fd8bba1, deleteOnExit=true 2024-11-10T15:16:03,823 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T15:16:03,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/test.cache.data in system properties and HBase conf 2024-11-10T15:16:03,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T15:16:03,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/hadoop.log.dir in system properties and HBase conf 2024-11-10T15:16:03,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T15:16:03,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T15:16:03,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T15:16:03,964 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T15:16:04,111 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T15:16:04,117 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T15:16:04,118 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T15:16:04,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T15:16:04,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T15:16:04,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T15:16:04,123 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T15:16:04,125 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T15:16:04,126 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T15:16:04,127 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T15:16:04,127 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/nfs.dump.dir in system properties and HBase conf 2024-11-10T15:16:04,128 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/java.io.tmpdir in system properties and HBase conf 2024-11-10T15:16:04,128 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T15:16:04,129 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T15:16:04,129 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T15:16:05,371 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-10T15:16:05,521 INFO [Time-limited test {}] log.Log(170): Logging initialized @3307ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-10T15:16:05,632 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T15:16:05,723 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T15:16:05,770 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T15:16:05,771 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T15:16:05,772 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T15:16:05,784 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T15:16:05,787 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/hadoop.log.dir/,AVAILABLE} 2024-11-10T15:16:05,788 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T15:16:05,973 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/java.io.tmpdir/jetty-localhost-41161-hadoop-hdfs-3_4_1-tests_jar-_-any-4297012956510600740/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T15:16:05,980 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:41161} 2024-11-10T15:16:05,980 INFO [Time-limited test {}] server.Server(415): Started @3768ms 2024-11-10T15:16:06,910 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T15:16:06,920 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T15:16:06,927 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T15:16:06,927 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T15:16:06,927 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T15:16:06,928 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/hadoop.log.dir/,AVAILABLE} 2024-11-10T15:16:06,929 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T15:16:07,057 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/java.io.tmpdir/jetty-localhost-38045-hadoop-hdfs-3_4_1-tests_jar-_-any-11118036895316310955/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T15:16:07,057 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:38045} 2024-11-10T15:16:07,058 INFO [Time-limited test {}] server.Server(415): Started @4845ms 2024-11-10T15:16:07,124 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T15:16:09,261 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/cluster_33e785d3-c89b-58a2-b49e-d3558fd8bba1/data/data1/current/BP-1898693680-172.17.0.3-1731251764747/current, will proceed with Du for space computation calculation, 2024-11-10T15:16:09,261 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/cluster_33e785d3-c89b-58a2-b49e-d3558fd8bba1/data/data2/current/BP-1898693680-172.17.0.3-1731251764747/current, will proceed with Du for space computation calculation, 2024-11-10T15:16:09,309 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T15:16:09,364 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x43bb9866950748f1 with lease ID 0x290058a010ce506c: Processing first storage report for DS-9ad34f0f-76b0-479f-b22e-ddf579965715 from datanode DatanodeRegistration(127.0.0.1:45567, datanodeUuid=a624e2af-477c-4cb8-aea7-386ba3eb7545, infoPort=40493, infoSecurePort=0, ipcPort=44795, storageInfo=lv=-57;cid=testClusterID;nsid=764303773;c=1731251764747) 2024-11-10T15:16:09,367 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x43bb9866950748f1 with lease ID 0x290058a010ce506c: from storage DS-9ad34f0f-76b0-479f-b22e-ddf579965715 node DatanodeRegistration(127.0.0.1:45567, datanodeUuid=a624e2af-477c-4cb8-aea7-386ba3eb7545, infoPort=40493, infoSecurePort=0, ipcPort=44795, storageInfo=lv=-57;cid=testClusterID;nsid=764303773;c=1731251764747), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-10T15:16:09,368 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x43bb9866950748f1 with lease ID 0x290058a010ce506c: Processing first storage report for DS-585b3918-5fcd-46c2-9a09-052480883895 from datanode DatanodeRegistration(127.0.0.1:45567, datanodeUuid=a624e2af-477c-4cb8-aea7-386ba3eb7545, infoPort=40493, infoSecurePort=0, ipcPort=44795, storageInfo=lv=-57;cid=testClusterID;nsid=764303773;c=1731251764747) 2024-11-10T15:16:09,368 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x43bb9866950748f1 with lease ID 0x290058a010ce506c: from storage DS-585b3918-5fcd-46c2-9a09-052480883895 node DatanodeRegistration(127.0.0.1:45567, datanodeUuid=a624e2af-477c-4cb8-aea7-386ba3eb7545, infoPort=40493, infoSecurePort=0, ipcPort=44795, storageInfo=lv=-57;cid=testClusterID;nsid=764303773;c=1731251764747), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T15:16:09,430 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d 2024-11-10T15:16:09,543 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/cluster_33e785d3-c89b-58a2-b49e-d3558fd8bba1/zookeeper_0, clientPort=59218, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/cluster_33e785d3-c89b-58a2-b49e-d3558fd8bba1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/cluster_33e785d3-c89b-58a2-b49e-d3558fd8bba1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T15:16:09,557 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59218 2024-11-10T15:16:09,571 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T15:16:09,574 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T15:16:09,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741825_1001 (size=7) 2024-11-10T15:16:10,272 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85 with version=8 2024-11-10T15:16:10,274 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/hbase-staging 2024-11-10T15:16:10,367 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T15:16:10,636 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c0771061be61:0 server-side Connection retries=45 2024-11-10T15:16:10,648 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T15:16:10,648 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T15:16:10,654 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T15:16:10,654 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T15:16:10,655 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T15:16:10,823 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T15:16:10,887 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T15:16:10,895 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T15:16:10,898 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T15:16:10,921 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8202 (auto-detected) 2024-11-10T15:16:10,922 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-10T15:16:10,939 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41501 2024-11-10T15:16:10,959 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41501 connecting to ZooKeeper ensemble=127.0.0.1:59218 2024-11-10T15:16:11,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:415010x0, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T15:16:11,045 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41501-0x10125518c520000 connected 2024-11-10T15:16:11,157 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T15:16:11,159 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T15:16:11,170 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T15:16:11,174 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85, hbase.cluster.distributed=false 2024-11-10T15:16:11,195 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T15:16:11,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41501 2024-11-10T15:16:11,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41501 2024-11-10T15:16:11,204 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41501 2024-11-10T15:16:11,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41501 2024-11-10T15:16:11,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41501 2024-11-10T15:16:11,337 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c0771061be61:0 server-side Connection retries=45 2024-11-10T15:16:11,339 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T15:16:11,340 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T15:16:11,340 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T15:16:11,340 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T15:16:11,341 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T15:16:11,348 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T15:16:11,352 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T15:16:11,356 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46087 2024-11-10T15:16:11,360 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46087 connecting to ZooKeeper ensemble=127.0.0.1:59218 2024-11-10T15:16:11,361 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T15:16:11,366 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T15:16:11,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:460870x0, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T15:16:11,378 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46087-0x10125518c520001 connected 2024-11-10T15:16:11,378 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T15:16:11,383 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T15:16:11,394 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T15:16:11,397 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T15:16:11,405 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T15:16:11,409 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46087 2024-11-10T15:16:11,409 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46087 2024-11-10T15:16:11,410 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46087 2024-11-10T15:16:11,414 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46087 2024-11-10T15:16:11,414 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46087 2024-11-10T15:16:11,433 DEBUG [M:0;c0771061be61:41501 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c0771061be61:41501 2024-11-10T15:16:11,434 INFO [master/c0771061be61:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c0771061be61,41501,1731251770419 2024-11-10T15:16:11,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T15:16:11,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T15:16:11,453 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c0771061be61,41501,1731251770419 2024-11-10T15:16:11,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T15:16:11,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T15:16:11,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T15:16:11,501 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T15:16:11,504 INFO [master/c0771061be61:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c0771061be61,41501,1731251770419 from backup master directory 2024-11-10T15:16:11,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c0771061be61,41501,1731251770419 2024-11-10T15:16:11,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T15:16:11,514 WARN [master/c0771061be61:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T15:16:11,515 INFO [master/c0771061be61:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c0771061be61,41501,1731251770419 2024-11-10T15:16:11,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T15:16:11,521 INFO [master/c0771061be61:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T15:16:11,523 INFO [master/c0771061be61:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T15:16:11,586 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/hbase.id] with ID: 58ceec01-b1f5-4676-9924-48a824d2685c 2024-11-10T15:16:11,586 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/.tmp/hbase.id 2024-11-10T15:16:11,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741826_1002 (size=42) 2024-11-10T15:16:12,000 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/.tmp/hbase.id]:[hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/hbase.id] 2024-11-10T15:16:12,058 INFO [master/c0771061be61:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T15:16:12,064 INFO [master/c0771061be61:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T15:16:12,086 INFO [master/c0771061be61:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-10T15:16:12,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T15:16:12,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T15:16:12,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741827_1003 (size=196) 2024-11-10T15:16:12,555 INFO [master/c0771061be61:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T15:16:12,558 INFO [master/c0771061be61:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T15:16:12,577 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:12,582 INFO [master/c0771061be61:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T15:16:12,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741828_1004 (size=1189) 2024-11-10T15:16:13,029 INFO [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store 2024-11-10T15:16:13,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741829_1005 (size=34) 2024-11-10T15:16:13,452 INFO [master/c0771061be61:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T15:16:13,457 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:16:13,458 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T15:16:13,459 INFO [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T15:16:13,459 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T15:16:13,461 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T15:16:13,462 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T15:16:13,462 INFO [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T15:16:13,464 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731251773458Disabling compacts and flushes for region at 1731251773458Disabling writes for close at 1731251773461 (+3 ms)Writing region close event to WAL at 1731251773462 (+1 ms)Closed at 1731251773462 2024-11-10T15:16:13,467 WARN [master/c0771061be61:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/.initializing 2024-11-10T15:16:13,467 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/WALs/c0771061be61,41501,1731251770419 2024-11-10T15:16:13,477 INFO [master/c0771061be61:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T15:16:13,499 INFO [master/c0771061be61:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c0771061be61%2C41501%2C1731251770419, suffix=, logDir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/WALs/c0771061be61,41501,1731251770419, archiveDir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/oldWALs, maxLogs=10 2024-11-10T15:16:13,533 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/WALs/c0771061be61,41501,1731251770419/c0771061be61%2C41501%2C1731251770419.1731251773506, exclude list is [], retry=0 2024-11-10T15:16:13,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45567,DS-9ad34f0f-76b0-479f-b22e-ddf579965715,DISK] 2024-11-10T15:16:13,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-10T15:16:13,618 INFO [master/c0771061be61:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/WALs/c0771061be61,41501,1731251770419/c0771061be61%2C41501%2C1731251770419.1731251773506 2024-11-10T15:16:13,623 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40493:40493)] 2024-11-10T15:16:13,624 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T15:16:13,625 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:16:13,629 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T15:16:13,631 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T15:16:13,678 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T15:16:13,708 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T15:16:13,712 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:13,715 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T15:16:13,716 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T15:16:13,720 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T15:16:13,721 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:13,722 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:16:13,722 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T15:16:13,725 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T15:16:13,725 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:13,726 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:16:13,726 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T15:16:13,729 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T15:16:13,729 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:13,731 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:16:13,731 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T15:16:13,735 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T15:16:13,737 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T15:16:13,746 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T15:16:13,747 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T15:16:13,754 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T15:16:13,758 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T15:16:13,762 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T15:16:13,763 INFO [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60272333, jitterRate=-0.10187225043773651}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T15:16:13,776 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731251773648Initializing all the Stores at 1731251773651 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731251773652 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251773653 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251773653Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251773653Cleaning up temporary data from old regions at 1731251773747 (+94 ms)Region opened successfully at 1731251773771 (+24 ms) 2024-11-10T15:16:13,778 INFO [master/c0771061be61:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T15:16:13,817 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f113b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c0771061be61/172.17.0.3:0 2024-11-10T15:16:13,861 INFO [master/c0771061be61:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T15:16:13,875 INFO [master/c0771061be61:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T15:16:13,876 INFO [master/c0771061be61:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T15:16:13,879 INFO [master/c0771061be61:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T15:16:13,880 INFO [master/c0771061be61:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-10T15:16:13,885 INFO [master/c0771061be61:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-10T15:16:13,885 INFO [master/c0771061be61:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T15:16:13,910 INFO [master/c0771061be61:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T15:16:13,921 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T15:16:13,934 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T15:16:13,936 INFO [master/c0771061be61:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T15:16:13,938 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T15:16:13,944 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T15:16:13,947 INFO [master/c0771061be61:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T15:16:13,951 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T15:16:13,955 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T15:16:13,957 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T15:16:13,965 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T15:16:13,989 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T15:16:14,007 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T15:16:14,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T15:16:14,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T15:16:14,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T15:16:14,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T15:16:14,033 INFO [master/c0771061be61:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c0771061be61,41501,1731251770419, sessionid=0x10125518c520000, setting cluster-up flag (Was=false) 2024-11-10T15:16:14,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T15:16:14,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T15:16:14,092 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T15:16:14,094 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c0771061be61,41501,1731251770419 2024-11-10T15:16:14,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T15:16:14,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T15:16:14,144 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T15:16:14,146 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c0771061be61,41501,1731251770419 2024-11-10T15:16:14,153 INFO [master/c0771061be61:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T15:16:14,221 INFO [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(746): ClusterId : 58ceec01-b1f5-4676-9924-48a824d2685c 2024-11-10T15:16:14,225 DEBUG [RS:0;c0771061be61:46087 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T15:16:14,224 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T15:16:14,235 INFO [master/c0771061be61:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T15:16:14,241 DEBUG [RS:0;c0771061be61:46087 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T15:16:14,241 DEBUG [RS:0;c0771061be61:46087 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T15:16:14,241 INFO [master/c0771061be61:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T15:16:14,252 DEBUG [RS:0;c0771061be61:46087 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T15:16:14,249 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c0771061be61,41501,1731251770419 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T15:16:14,253 DEBUG [RS:0;c0771061be61:46087 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22fba83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c0771061be61/172.17.0.3:0 2024-11-10T15:16:14,260 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c0771061be61:0, corePoolSize=5, maxPoolSize=5 2024-11-10T15:16:14,261 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c0771061be61:0, corePoolSize=5, maxPoolSize=5 2024-11-10T15:16:14,261 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c0771061be61:0, corePoolSize=5, maxPoolSize=5 2024-11-10T15:16:14,261 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c0771061be61:0, corePoolSize=5, maxPoolSize=5 2024-11-10T15:16:14,262 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c0771061be61:0, corePoolSize=10, maxPoolSize=10 2024-11-10T15:16:14,262 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c0771061be61:0, corePoolSize=1, maxPoolSize=1 2024-11-10T15:16:14,262 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c0771061be61:0, corePoolSize=2, maxPoolSize=2 2024-11-10T15:16:14,262 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c0771061be61:0, corePoolSize=1, maxPoolSize=1 2024-11-10T15:16:14,273 DEBUG [RS:0;c0771061be61:46087 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c0771061be61:46087 2024-11-10T15:16:14,278 INFO [RS:0;c0771061be61:46087 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T15:16:14,278 INFO [RS:0;c0771061be61:46087 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T15:16:14,278 DEBUG [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T15:16:14,279 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T15:16:14,280 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T15:16:14,282 INFO [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(2659): reportForDuty to master=c0771061be61,41501,1731251770419 with port=46087, startcode=1731251771284 2024-11-10T15:16:14,284 INFO [master/c0771061be61:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731251804283 2024-11-10T15:16:14,286 INFO [master/c0771061be61:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T15:16:14,286 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:14,286 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T15:16:14,287 INFO [master/c0771061be61:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T15:16:14,290 INFO [master/c0771061be61:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T15:16:14,291 INFO [master/c0771061be61:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T15:16:14,291 INFO [master/c0771061be61:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T15:16:14,291 INFO [master/c0771061be61:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T15:16:14,294 DEBUG [RS:0;c0771061be61:46087 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T15:16:14,292 INFO [master/c0771061be61:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:14,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741831_1007 (size=1321) 2024-11-10T15:16:14,301 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T15:16:14,302 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85 2024-11-10T15:16:14,303 INFO [master/c0771061be61:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T15:16:14,307 INFO [master/c0771061be61:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T15:16:14,308 INFO [master/c0771061be61:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T15:16:14,311 INFO [master/c0771061be61:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T15:16:14,312 INFO [master/c0771061be61:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T15:16:14,314 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c0771061be61:0:becomeActiveMaster-HFileCleaner.large.0-1731251774313,5,FailOnTimeoutGroup] 2024-11-10T15:16:14,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741832_1008 (size=32) 2024-11-10T15:16:14,319 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:16:14,324 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c0771061be61:0:becomeActiveMaster-HFileCleaner.small.0-1731251774314,5,FailOnTimeoutGroup] 2024-11-10T15:16:14,324 INFO [master/c0771061be61:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:14,325 INFO [master/c0771061be61:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T15:16:14,325 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T15:16:14,326 INFO [master/c0771061be61:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:14,326 INFO [master/c0771061be61:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:14,329 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T15:16:14,330 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:14,332 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T15:16:14,333 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T15:16:14,336 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T15:16:14,336 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:14,337 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55087, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T15:16:14,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T15:16:14,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T15:16:14,341 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T15:16:14,341 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:14,342 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T15:16:14,343 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T15:16:14,344 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41501 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c0771061be61,46087,1731251771284 2024-11-10T15:16:14,345 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T15:16:14,345 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:14,346 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41501 {}] master.ServerManager(517): Registering regionserver=c0771061be61,46087,1731251771284 2024-11-10T15:16:14,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T15:16:14,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T15:16:14,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740 2024-11-10T15:16:14,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740 2024-11-10T15:16:14,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T15:16:14,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T15:16:14,354 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T15:16:14,357 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T15:16:14,360 DEBUG [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85 2024-11-10T15:16:14,360 DEBUG [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46879 2024-11-10T15:16:14,360 DEBUG [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T15:16:14,361 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T15:16:14,362 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59105254, jitterRate=-0.11926308274269104}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T15:16:14,366 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731251774319Initializing all the Stores at 1731251774323 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731251774323Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731251774325 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251774325Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731251774325Cleaning up temporary data from old regions at 1731251774352 (+27 ms)Region opened successfully at 1731251774366 (+14 ms) 2024-11-10T15:16:14,366 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T15:16:14,366 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T15:16:14,366 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T15:16:14,366 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T15:16:14,366 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T15:16:14,368 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T15:16:14,368 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731251774366Disabling compacts and flushes for region at 1731251774366Disabling writes for close at 1731251774366Writing region close event to WAL at 1731251774367 (+1 ms)Closed at 1731251774368 (+1 ms) 2024-11-10T15:16:14,371 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T15:16:14,371 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T15:16:14,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T15:16:14,376 DEBUG [RS:0;c0771061be61:46087 {}] zookeeper.ZKUtil(111): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c0771061be61,46087,1731251771284 2024-11-10T15:16:14,376 WARN [RS:0;c0771061be61:46087 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T15:16:14,377 INFO [RS:0;c0771061be61:46087 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T15:16:14,377 DEBUG [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/WALs/c0771061be61,46087,1731251771284 2024-11-10T15:16:14,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T15:16:14,379 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c0771061be61,46087,1731251771284] 2024-11-10T15:16:14,387 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T15:16:14,390 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T15:16:14,402 INFO [RS:0;c0771061be61:46087 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T15:16:14,419 INFO [RS:0;c0771061be61:46087 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T15:16:14,424 INFO [RS:0;c0771061be61:46087 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T15:16:14,424 INFO [RS:0;c0771061be61:46087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:14,425 INFO [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T15:16:14,430 INFO [RS:0;c0771061be61:46087 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T15:16:14,432 INFO [RS:0;c0771061be61:46087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:14,432 DEBUG [RS:0;c0771061be61:46087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c0771061be61:0, corePoolSize=1, maxPoolSize=1 2024-11-10T15:16:14,432 DEBUG [RS:0;c0771061be61:46087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c0771061be61:0, corePoolSize=1, maxPoolSize=1 2024-11-10T15:16:14,432 DEBUG [RS:0;c0771061be61:46087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c0771061be61:0, corePoolSize=1, maxPoolSize=1 2024-11-10T15:16:14,432 DEBUG [RS:0;c0771061be61:46087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c0771061be61:0, corePoolSize=1, maxPoolSize=1 2024-11-10T15:16:14,433 DEBUG [RS:0;c0771061be61:46087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c0771061be61:0, corePoolSize=1, maxPoolSize=1 2024-11-10T15:16:14,433 DEBUG [RS:0;c0771061be61:46087 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c0771061be61:0, corePoolSize=2, maxPoolSize=2 2024-11-10T15:16:14,433 DEBUG [RS:0;c0771061be61:46087 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0, corePoolSize=1, maxPoolSize=1 2024-11-10T15:16:14,433 DEBUG [RS:0;c0771061be61:46087 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c0771061be61:0, corePoolSize=1, maxPoolSize=1 2024-11-10T15:16:14,433 DEBUG [RS:0;c0771061be61:46087 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c0771061be61:0, corePoolSize=1, maxPoolSize=1 2024-11-10T15:16:14,433 DEBUG [RS:0;c0771061be61:46087 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c0771061be61:0, corePoolSize=1, maxPoolSize=1 2024-11-10T15:16:14,433 DEBUG [RS:0;c0771061be61:46087 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c0771061be61:0, corePoolSize=1, maxPoolSize=1 2024-11-10T15:16:14,433 DEBUG [RS:0;c0771061be61:46087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c0771061be61:0, corePoolSize=1, maxPoolSize=1 2024-11-10T15:16:14,434 DEBUG [RS:0;c0771061be61:46087 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c0771061be61:0, corePoolSize=3, maxPoolSize=3 2024-11-10T15:16:14,434 DEBUG [RS:0;c0771061be61:46087 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0, corePoolSize=3, maxPoolSize=3 2024-11-10T15:16:14,435 INFO [RS:0;c0771061be61:46087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:14,435 INFO [RS:0;c0771061be61:46087 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:14,435 INFO [RS:0;c0771061be61:46087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:14,435 INFO [RS:0;c0771061be61:46087 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:14,435 INFO [RS:0;c0771061be61:46087 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:14,436 INFO [RS:0;c0771061be61:46087 {}] hbase.ChoreService(168): Chore ScheduledChore name=c0771061be61,46087,1731251771284-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T15:16:14,453 INFO [RS:0;c0771061be61:46087 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T15:16:14,455 INFO [RS:0;c0771061be61:46087 {}] hbase.ChoreService(168): Chore ScheduledChore name=c0771061be61,46087,1731251771284-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:14,455 INFO [RS:0;c0771061be61:46087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:14,455 INFO [RS:0;c0771061be61:46087 {}] regionserver.Replication(171): c0771061be61,46087,1731251771284 started 2024-11-10T15:16:14,473 INFO [RS:0;c0771061be61:46087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:14,473 INFO [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(1482): Serving as c0771061be61,46087,1731251771284, RpcServer on c0771061be61/172.17.0.3:46087, sessionid=0x10125518c520001 2024-11-10T15:16:14,474 DEBUG [RS:0;c0771061be61:46087 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T15:16:14,475 DEBUG [RS:0;c0771061be61:46087 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c0771061be61,46087,1731251771284 2024-11-10T15:16:14,475 DEBUG [RS:0;c0771061be61:46087 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c0771061be61,46087,1731251771284' 2024-11-10T15:16:14,475 DEBUG [RS:0;c0771061be61:46087 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T15:16:14,476 DEBUG [RS:0;c0771061be61:46087 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T15:16:14,477 DEBUG [RS:0;c0771061be61:46087 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T15:16:14,477 DEBUG [RS:0;c0771061be61:46087 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T15:16:14,478 DEBUG [RS:0;c0771061be61:46087 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c0771061be61,46087,1731251771284 2024-11-10T15:16:14,478 DEBUG [RS:0;c0771061be61:46087 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c0771061be61,46087,1731251771284' 2024-11-10T15:16:14,478 DEBUG [RS:0;c0771061be61:46087 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T15:16:14,479 DEBUG [RS:0;c0771061be61:46087 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T15:16:14,479 DEBUG [RS:0;c0771061be61:46087 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T15:16:14,479 INFO [RS:0;c0771061be61:46087 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T15:16:14,480 INFO [RS:0;c0771061be61:46087 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T15:16:14,541 WARN [c0771061be61:41501 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-10T15:16:14,585 INFO [RS:0;c0771061be61:46087 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T15:16:14,588 INFO [RS:0;c0771061be61:46087 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c0771061be61%2C46087%2C1731251771284, suffix=, logDir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/WALs/c0771061be61,46087,1731251771284, archiveDir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/oldWALs, maxLogs=32 2024-11-10T15:16:14,606 DEBUG [RS:0;c0771061be61:46087 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/WALs/c0771061be61,46087,1731251771284/c0771061be61%2C46087%2C1731251771284.1731251774592, exclude list is [], retry=0 2024-11-10T15:16:14,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45567,DS-9ad34f0f-76b0-479f-b22e-ddf579965715,DISK] 2024-11-10T15:16:14,614 INFO [RS:0;c0771061be61:46087 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/WALs/c0771061be61,46087,1731251771284/c0771061be61%2C46087%2C1731251771284.1731251774592 2024-11-10T15:16:14,615 DEBUG [RS:0;c0771061be61:46087 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40493:40493)] 2024-11-10T15:16:14,793 DEBUG [c0771061be61:41501 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T15:16:14,804 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:16:14,809 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c0771061be61,46087,1731251771284, state=OPENING 2024-11-10T15:16:14,828 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T15:16:14,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T15:16:14,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T15:16:14,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T15:16:14,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T15:16:14,863 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T15:16:14,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:16:15,046 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T15:16:15,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36029, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T15:16:15,060 INFO [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T15:16:15,060 INFO [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T15:16:15,061 INFO [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T15:16:15,064 INFO [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c0771061be61%2C46087%2C1731251771284.meta, suffix=.meta, logDir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/WALs/c0771061be61,46087,1731251771284, archiveDir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/oldWALs, maxLogs=32 2024-11-10T15:16:15,079 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/WALs/c0771061be61,46087,1731251771284/c0771061be61%2C46087%2C1731251771284.meta.1731251775066.meta, exclude list is [], retry=0 2024-11-10T15:16:15,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45567,DS-9ad34f0f-76b0-479f-b22e-ddf579965715,DISK] 2024-11-10T15:16:15,086 INFO [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/WALs/c0771061be61,46087,1731251771284/c0771061be61%2C46087%2C1731251771284.meta.1731251775066.meta 2024-11-10T15:16:15,087 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40493:40493)] 2024-11-10T15:16:15,087 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T15:16:15,089 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T15:16:15,092 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T15:16:15,097 INFO [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T15:16:15,101 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T15:16:15,102 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:16:15,102 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T15:16:15,102 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T15:16:15,105 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T15:16:15,106 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T15:16:15,107 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:15,107 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T15:16:15,108 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T15:16:15,109 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T15:16:15,109 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:15,110 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T15:16:15,110 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T15:16:15,111 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T15:16:15,111 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:15,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T15:16:15,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T15:16:15,114 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T15:16:15,114 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:15,114 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T15:16:15,115 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T15:16:15,116 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740 2024-11-10T15:16:15,119 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740 2024-11-10T15:16:15,121 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T15:16:15,121 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T15:16:15,122 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T15:16:15,125 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T15:16:15,127 INFO [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72769957, jitterRate=0.08435685932636261}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T15:16:15,127 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T15:16:15,128 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731251775103Writing region info on filesystem at 1731251775103Initializing all the Stores at 1731251775105 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731251775105Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731251775105Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251775105Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731251775105Cleaning up temporary data from old regions at 1731251775121 (+16 ms)Running coprocessor post-open hooks at 1731251775127 (+6 ms)Region opened successfully at 1731251775128 (+1 ms) 2024-11-10T15:16:15,136 INFO [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731251775034 2024-11-10T15:16:15,148 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:16:15,150 DEBUG [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T15:16:15,151 INFO [RS_OPEN_META-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T15:16:15,151 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c0771061be61,46087,1731251771284, state=OPEN 2024-11-10T15:16:15,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T15:16:15,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T15:16:15,230 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T15:16:15,230 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T15:16:15,231 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:15,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T15:16:15,241 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c0771061be61,46087,1731251771284 in 366 msec 2024-11-10T15:16:15,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T15:16:15,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 865 msec 2024-11-10T15:16:15,252 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T15:16:15,252 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T15:16:15,274 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:15,276 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:15,295 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:15,297 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45201, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:15,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1360 sec 2024-11-10T15:16:15,319 INFO [master/c0771061be61:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731251775319, completionTime=-1 2024-11-10T15:16:15,323 INFO [master/c0771061be61:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T15:16:15,323 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T15:16:15,354 INFO [master/c0771061be61:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T15:16:15,354 INFO [master/c0771061be61:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731251835354 2024-11-10T15:16:15,354 INFO [master/c0771061be61:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731251895354 2024-11-10T15:16:15,354 INFO [master/c0771061be61:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 31 msec 2024-11-10T15:16:15,356 INFO [master/c0771061be61:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0771061be61,41501,1731251770419-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:15,356 INFO [master/c0771061be61:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0771061be61,41501,1731251770419-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:15,357 INFO [master/c0771061be61:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0771061be61,41501,1731251770419-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:15,358 INFO [master/c0771061be61:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c0771061be61:41501, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:15,358 INFO [master/c0771061be61:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:15,359 INFO [master/c0771061be61:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:15,365 DEBUG [master/c0771061be61:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T15:16:15,401 INFO [master/c0771061be61:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.886sec 2024-11-10T15:16:15,402 INFO [master/c0771061be61:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T15:16:15,403 INFO [master/c0771061be61:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T15:16:15,404 INFO [master/c0771061be61:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T15:16:15,405 INFO [master/c0771061be61:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T15:16:15,405 INFO [master/c0771061be61:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T15:16:15,406 INFO [master/c0771061be61:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0771061be61,41501,1731251770419-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T15:16:15,406 INFO [master/c0771061be61:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0771061be61,41501,1731251770419-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T15:16:15,414 DEBUG [master/c0771061be61:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T15:16:15,415 INFO [master/c0771061be61:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T15:16:15,415 INFO [master/c0771061be61:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0771061be61,41501,1731251770419-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T15:16:15,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4346ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:15,433 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T15:16:15,433 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T15:16:15,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:15,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:15,446 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:15,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bece0d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:15,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:15,455 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:15,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bbb1cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:15,458 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:15,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:15,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:15,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:15,487 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c0771061be61,41501,1731251770419 2024-11-10T15:16:15,499 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=585, ProcessCount=11, AvailableMemoryMB=4239 2024-11-10T15:16:15,507 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T15:16:15,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c0771061be61,41501,1731251770419 2024-11-10T15:16:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56ef0928 2024-11-10T15:16:15,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T15:16:15,519 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T15:16:15,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T15:16:15,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T15:16:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T15:16:15,537 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T15:16:15,539 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-10T15:16:15,540 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:15,543 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T15:16:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T15:16:15,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741835_1011 (size=1061) 2024-11-10T15:16:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T15:16:15,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T15:16:15,959 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85 2024-11-10T15:16:15,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741836_1012 (size=53) 2024-11-10T15:16:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T15:16:16,377 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:16:16,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6bf704d88e72898d090c362728826653, disabling compactions & flushes 2024-11-10T15:16:16,378 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:16,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:16,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. after waiting 0 ms 2024-11-10T15:16:16,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:16,378 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:16,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6bf704d88e72898d090c362728826653: Waiting for close lock at 1731251776377Disabling compacts and flushes for region at 1731251776377Disabling writes for close at 1731251776378 (+1 ms)Writing region close event to WAL at 1731251776378Closed at 1731251776378 2024-11-10T15:16:16,381 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T15:16:16,386 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731251776381"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731251776381"}]},"ts":"1731251776381"} 2024-11-10T15:16:16,392 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T15:16:16,395 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T15:16:16,398 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251776395"}]},"ts":"1731251776395"} 2024-11-10T15:16:16,403 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T15:16:16,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bf704d88e72898d090c362728826653, ASSIGN}] 2024-11-10T15:16:16,408 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bf704d88e72898d090c362728826653, ASSIGN 2024-11-10T15:16:16,411 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bf704d88e72898d090c362728826653, ASSIGN; state=OFFLINE, location=c0771061be61,46087,1731251771284; forceNewPlan=false, retain=false 2024-11-10T15:16:16,563 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6bf704d88e72898d090c362728826653, regionState=OPENING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:16:16,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bf704d88e72898d090c362728826653, ASSIGN because future has completed 2024-11-10T15:16:16,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:16:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T15:16:16,745 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:16,745 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} 2024-11-10T15:16:16,746 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6bf704d88e72898d090c362728826653 2024-11-10T15:16:16,746 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:16:16,746 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6bf704d88e72898d090c362728826653 2024-11-10T15:16:16,746 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6bf704d88e72898d090c362728826653 2024-11-10T15:16:16,749 INFO [StoreOpener-6bf704d88e72898d090c362728826653-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6bf704d88e72898d090c362728826653 2024-11-10T15:16:16,753 INFO [StoreOpener-6bf704d88e72898d090c362728826653-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:16:16,754 INFO [StoreOpener-6bf704d88e72898d090c362728826653-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bf704d88e72898d090c362728826653 columnFamilyName A 2024-11-10T15:16:16,754 DEBUG [StoreOpener-6bf704d88e72898d090c362728826653-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:16,756 INFO [StoreOpener-6bf704d88e72898d090c362728826653-1 {}] regionserver.HStore(327): Store=6bf704d88e72898d090c362728826653/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:16:16,756 INFO [StoreOpener-6bf704d88e72898d090c362728826653-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6bf704d88e72898d090c362728826653 2024-11-10T15:16:16,759 INFO [StoreOpener-6bf704d88e72898d090c362728826653-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:16:16,759 INFO [StoreOpener-6bf704d88e72898d090c362728826653-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bf704d88e72898d090c362728826653 columnFamilyName B 2024-11-10T15:16:16,759 DEBUG [StoreOpener-6bf704d88e72898d090c362728826653-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:16,760 INFO [StoreOpener-6bf704d88e72898d090c362728826653-1 {}] regionserver.HStore(327): Store=6bf704d88e72898d090c362728826653/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:16:16,761 INFO [StoreOpener-6bf704d88e72898d090c362728826653-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6bf704d88e72898d090c362728826653 2024-11-10T15:16:16,763 INFO [StoreOpener-6bf704d88e72898d090c362728826653-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:16:16,764 INFO [StoreOpener-6bf704d88e72898d090c362728826653-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bf704d88e72898d090c362728826653 columnFamilyName C 2024-11-10T15:16:16,764 DEBUG [StoreOpener-6bf704d88e72898d090c362728826653-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:16,765 INFO [StoreOpener-6bf704d88e72898d090c362728826653-1 {}] regionserver.HStore(327): Store=6bf704d88e72898d090c362728826653/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:16:16,765 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:16,765 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6bf704d88e72898d090c362728826653 2024-11-10T15:16:16,767 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653 2024-11-10T15:16:16,768 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653 2024-11-10T15:16:16,771 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6bf704d88e72898d090c362728826653 2024-11-10T15:16:16,771 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6bf704d88e72898d090c362728826653 2024-11-10T15:16:16,772 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T15:16:16,776 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6bf704d88e72898d090c362728826653 2024-11-10T15:16:16,786 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T15:16:16,787 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6bf704d88e72898d090c362728826653; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58929501, jitterRate=-0.12188200652599335}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T15:16:16,788 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6bf704d88e72898d090c362728826653 2024-11-10T15:16:16,789 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6bf704d88e72898d090c362728826653: Running coprocessor pre-open hook at 1731251776746Writing region info on filesystem at 1731251776746Initializing all the Stores at 1731251776748 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251776748Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251776749 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251776749Cleaning up temporary data from old regions at 1731251776771 (+22 ms)Running coprocessor post-open hooks at 1731251776788 (+17 ms)Region opened successfully at 1731251776788 2024-11-10T15:16:16,791 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., pid=6, masterSystemTime=1731251776732 2024-11-10T15:16:16,796 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:16,797 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:16,799 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6bf704d88e72898d090c362728826653, regionState=OPEN, openSeqNum=2, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:16:16,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:16:16,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T15:16:16,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 in 235 msec 2024-11-10T15:16:16,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T15:16:16,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bf704d88e72898d090c362728826653, ASSIGN in 416 msec 2024-11-10T15:16:16,838 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T15:16:16,838 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251776838"}]},"ts":"1731251776838"} 2024-11-10T15:16:16,843 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T15:16:16,846 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T15:16:16,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3170 sec 2024-11-10T15:16:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T15:16:17,693 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:17,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2c994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,700 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:17,700 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:17,702 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46cd3e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:17,706 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:17,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c82fed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,709 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:17,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:17,717 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T15:16:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c0771061be61,41501,1731251770419 2024-11-10T15:16:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@da7d497 2024-11-10T15:16:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T15:16:17,724 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T15:16:17,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:17,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:17,733 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@483a22c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:17,736 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63181207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,740 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:17,740 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:17,742 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:17,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d22a118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:17,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:17,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:17,745 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,748 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:17,748 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:17,749 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:17,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:17,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a6e4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:17,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:17,753 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef39c97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,756 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:17,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:17,759 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:17,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395f1c25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:17,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:17,766 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:17,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7ca9c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,769 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:17,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:17,771 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:17,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:17,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4282c07e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:17,774 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:17,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25318c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,777 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:17,777 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:17,778 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:17,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3c1f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:17,782 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:17,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e1b379, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,784 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:17,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:17,786 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:17,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:17,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:17,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d111af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:17,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:17,790 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:17,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e9639cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,798 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:17,799 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:17,800 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e406351, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:17,804 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d22a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,810 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:17,810 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:17,811 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:17,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:17,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:17,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c994daa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:17,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:17,815 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e186e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:17,832 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:17,834 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:17,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:17,835 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:17,837 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:17,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:17,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:17,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:17,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:17,846 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:17,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:17,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:16:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:16:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:16:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:16:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:16:17,865 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:17,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:17,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:17,878 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:16:17,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:16:17,889 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:17,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:17,898 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:16:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-10T15:16:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T15:16:17,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:17,911 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:16:17,914 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:16:17,917 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:17,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:16:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:17,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:16:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:16:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T15:16:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:18,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T15:16:18,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:18,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:18,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:18,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:18,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:18,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:18,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T15:16:18,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:18,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:18,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:18,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/d4aefe8afc9c4bc78bd47b77ff1c364e is 50, key is test_row_0/A:col10/1731251777917/Put/seqid=0 2024-11-10T15:16:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T15:16:18,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741837_1013 (size=12001) 2024-11-10T15:16:18,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/d4aefe8afc9c4bc78bd47b77ff1c364e 2024-11-10T15:16:18,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251838189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:18,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251838198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:18,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251838199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:18,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251838215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:18,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37946 deadline: 1731251838221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:18,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T15:16:18,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:18,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:18,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:18,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/e6e7563283c44bb39c338fc031bd8fc1 is 50, key is test_row_0/B:col10/1731251777917/Put/seqid=0 2024-11-10T15:16:18,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37946 deadline: 1731251838424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:18,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251838424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:18,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741838_1014 (size=12001) 2024-11-10T15:16:18,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251838424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:18,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/e6e7563283c44bb39c338fc031bd8fc1 2024-11-10T15:16:18,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:18,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251838425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:18,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251838433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:18,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:18,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T15:16:18,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:18,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:18,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:18,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/4bf2a236eaa246bda03b4bce4c11f38a is 50, key is test_row_0/C:col10/1731251777917/Put/seqid=0 2024-11-10T15:16:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T15:16:18,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741839_1015 (size=12001) 2024-11-10T15:16:18,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/4bf2a236eaa246bda03b4bce4c11f38a 2024-11-10T15:16:18,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/d4aefe8afc9c4bc78bd47b77ff1c364e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/d4aefe8afc9c4bc78bd47b77ff1c364e 2024-11-10T15:16:18,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/d4aefe8afc9c4bc78bd47b77ff1c364e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T15:16:18,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/e6e7563283c44bb39c338fc031bd8fc1 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e6e7563283c44bb39c338fc031bd8fc1 2024-11-10T15:16:18,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e6e7563283c44bb39c338fc031bd8fc1, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T15:16:18,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/4bf2a236eaa246bda03b4bce4c11f38a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/4bf2a236eaa246bda03b4bce4c11f38a 2024-11-10T15:16:18,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37946 deadline: 1731251838643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:18,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/4bf2a236eaa246bda03b4bce4c11f38a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T15:16:18,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T15:16:18,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:18,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:18,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:18,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6bf704d88e72898d090c362728826653 in 638ms, sequenceid=13, compaction requested=false 2024-11-10T15:16:18,678 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T15:16:18,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:18,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251778180, max timestamp=1731251778675Num uniques 859; 2024-11-10T15:16:18,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:16:18,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-10T15:16:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:18,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:18,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251778180, max timestamp=1731251778687Num uniques 859; 2024-11-10T15:16:18,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:16:18,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:16:18,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:18,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:16:18,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:18,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:16:18,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/552b1505c53e4c328428dc23ed868a04 is 50, key is test_row_0/A:col10/1731251778695/Put/seqid=0 2024-11-10T15:16:18,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741840_1016 (size=12001) 2024-11-10T15:16:18,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/552b1505c53e4c328428dc23ed868a04 2024-11-10T15:16:18,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T15:16:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:18,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/05fbd66649854448a4ef4bbd7015cbe8 is 50, key is test_row_0/B:col10/1731251778695/Put/seqid=0 2024-11-10T15:16:18,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741841_1017 (size=12001) 2024-11-10T15:16:18,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/05fbd66649854448a4ef4bbd7015cbe8 2024-11-10T15:16:18,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/9c19487621ad4b95ae57d98cc1af0267 is 50, key is test_row_0/C:col10/1731251778695/Put/seqid=0 2024-11-10T15:16:18,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T15:16:18,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:18,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:18,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:18,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251838926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:18,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251838950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:18,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37946 deadline: 1731251838953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741842_1018 (size=12001) 2024-11-10T15:16:19,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251838982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251838988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T15:16:19,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251839102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251839102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251839133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251839144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T15:16:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:19,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:19,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:19,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:19,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:19,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T15:16:19,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:19,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:19,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:19,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:19,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:19,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251839343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251839353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251839353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251839354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/9c19487621ad4b95ae57d98cc1af0267 2024-11-10T15:16:19,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/552b1505c53e4c328428dc23ed868a04 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/552b1505c53e4c328428dc23ed868a04 2024-11-10T15:16:19,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/552b1505c53e4c328428dc23ed868a04, entries=150, sequenceid=36, filesize=11.7 K 2024-11-10T15:16:19,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/05fbd66649854448a4ef4bbd7015cbe8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/05fbd66649854448a4ef4bbd7015cbe8 2024-11-10T15:16:19,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T15:16:19,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:19,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:19,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:19,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:19,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:19,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/05fbd66649854448a4ef4bbd7015cbe8, entries=150, sequenceid=36, filesize=11.7 K 2024-11-10T15:16:19,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/9c19487621ad4b95ae57d98cc1af0267 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/9c19487621ad4b95ae57d98cc1af0267 2024-11-10T15:16:19,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37946 deadline: 1731251839512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/9c19487621ad4b95ae57d98cc1af0267, entries=150, sequenceid=36, filesize=11.7 K 2024-11-10T15:16:19,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~203.97 KB/208864, currentSize=114.05 KB/116790 for 6bf704d88e72898d090c362728826653 in 837ms, sequenceid=36, compaction requested=false 2024-11-10T15:16:19,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:19,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T15:16:19,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:19,634 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T15:16:19,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:19,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:19,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:19,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:19,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:19,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:19,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/472ddaed4e524745a591d7fd79b21d1b is 50, key is test_row_0/A:col10/1731251778901/Put/seqid=0 2024-11-10T15:16:19,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:19,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:19,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741843_1019 (size=12001) 2024-11-10T15:16:19,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251839742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251839742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251839744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251839766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251839873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251839873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251839873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:19,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251839882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:19,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:19,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T15:16:20,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251840087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:20,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251840092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:20,101 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/472ddaed4e524745a591d7fd79b21d1b 2024-11-10T15:16:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:20,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251840099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:20,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:20,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251840103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:20,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:20,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/c341d4df414b4662ac89105376bda5a9 is 50, key is test_row_0/B:col10/1731251778901/Put/seqid=0 2024-11-10T15:16:20,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741844_1020 (size=12001) 2024-11-10T15:16:20,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251840404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:20,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251840413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:20,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251840413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:20,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:20,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251840422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:20,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:20,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:20,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37946 deadline: 1731251840544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:20,559 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_2' in 6bf704d88e72898d090c362728826653 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2345 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:16:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:20,563 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/c341d4df414b4662ac89105376bda5a9 2024-11-10T15:16:20,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/856bdef5194e40b7ade42f1330465a5f is 50, key is test_row_0/C:col10/1731251778901/Put/seqid=0 2024-11-10T15:16:20,614 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T15:16:20,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741845_1021 (size=12001) 2024-11-10T15:16:20,644 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/856bdef5194e40b7ade42f1330465a5f 2024-11-10T15:16:20,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/472ddaed4e524745a591d7fd79b21d1b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/472ddaed4e524745a591d7fd79b21d1b 2024-11-10T15:16:20,721 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/472ddaed4e524745a591d7fd79b21d1b, entries=150, sequenceid=56, filesize=11.7 K 2024-11-10T15:16:20,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/c341d4df414b4662ac89105376bda5a9 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/c341d4df414b4662ac89105376bda5a9 2024-11-10T15:16:20,755 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/c341d4df414b4662ac89105376bda5a9, entries=150, sequenceid=56, filesize=11.7 K 2024-11-10T15:16:20,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/856bdef5194e40b7ade42f1330465a5f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/856bdef5194e40b7ade42f1330465a5f 2024-11-10T15:16:20,769 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T15:16:20,778 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/856bdef5194e40b7ade42f1330465a5f, entries=150, sequenceid=56, filesize=11.7 K 2024-11-10T15:16:20,787 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6bf704d88e72898d090c362728826653 in 1153ms, sequenceid=56, compaction requested=true 2024-11-10T15:16:20,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:20,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:20,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-10T15:16:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-10T15:16:20,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T15:16:20,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8800 sec 2024-11-10T15:16:20,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.9290 sec 2024-11-10T15:16:20,885 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T15:16:20,885 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T15:16:20,888 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T15:16:20,888 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T15:16:20,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T15:16:20,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T15:16:20,890 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T15:16:20,890 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T15:16:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:20,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T15:16:20,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:20,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:20,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:20,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:20,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:20,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:20,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/fe1268fb1bca4b09ab50093ffb20b261 is 50, key is test_row_0/A:col10/1731251780939/Put/seqid=0 2024-11-10T15:16:21,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741846_1022 (size=14341) 2024-11-10T15:16:21,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/fe1268fb1bca4b09ab50093ffb20b261 2024-11-10T15:16:21,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251841037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:21,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251841042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:21,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:21,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251841050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:21,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251841058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:21,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/dc290940265f49eaa6aec21344040e34 is 50, key is test_row_0/B:col10/1731251780939/Put/seqid=0 2024-11-10T15:16:21,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741847_1023 (size=12001) 2024-11-10T15:16:21,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251841182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:21,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:21,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251841185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:21,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:21,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251841191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:21,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:21,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:21,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251841191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:21,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251841393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:21,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251841393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:21,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:21,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:21,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251841402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:21,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251841402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:21,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/dc290940265f49eaa6aec21344040e34 2024-11-10T15:16:21,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/7f0c6acc871949cb8f89aa7355ea82dd is 50, key is test_row_0/C:col10/1731251780939/Put/seqid=0 2024-11-10T15:16:21,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741848_1024 (size=12001) 2024-11-10T15:16:21,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/7f0c6acc871949cb8f89aa7355ea82dd 2024-11-10T15:16:21,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/fe1268fb1bca4b09ab50093ffb20b261 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/fe1268fb1bca4b09ab50093ffb20b261 2024-11-10T15:16:21,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/fe1268fb1bca4b09ab50093ffb20b261, entries=200, sequenceid=69, filesize=14.0 K 2024-11-10T15:16:21,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/dc290940265f49eaa6aec21344040e34 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/dc290940265f49eaa6aec21344040e34 2024-11-10T15:16:21,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251841712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:21,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251841705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:21,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:21,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/dc290940265f49eaa6aec21344040e34, entries=150, sequenceid=69, filesize=11.7 K 2024-11-10T15:16:21,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/7f0c6acc871949cb8f89aa7355ea82dd as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/7f0c6acc871949cb8f89aa7355ea82dd 2024-11-10T15:16:21,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/7f0c6acc871949cb8f89aa7355ea82dd, entries=150, sequenceid=69, filesize=11.7 K 2024-11-10T15:16:21,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 6bf704d88e72898d090c362728826653 in 828ms, sequenceid=69, compaction requested=true 2024-11-10T15:16:21,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:21,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251781030, max timestamp=1731251781758Num uniques 859; 2024-11-10T15:16:21,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:16:21,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251781030, max timestamp=1731251781758Num uniques 859; 2024-11-10T15:16:21,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:16:21,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:16:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:21,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:21,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:16:21,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:21,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:16:21,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T15:16:21,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-10T15:16:21,795 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:16:21,796 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:16:21,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:21,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:16:21,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:21,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:16:21,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:21,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:16:21,809 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:16:21,811 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/B is initiating minor compaction (all files) 2024-11-10T15:16:21,816 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/B in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:21,817 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e6e7563283c44bb39c338fc031bd8fc1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/05fbd66649854448a4ef4bbd7015cbe8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/c341d4df414b4662ac89105376bda5a9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/dc290940265f49eaa6aec21344040e34] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=46.9 K 2024-11-10T15:16:21,819 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting e6e7563283c44bb39c338fc031bd8fc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731251777917 2024-11-10T15:16:21,820 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 05fbd66649854448a4ef4bbd7015cbe8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731251778196 2024-11-10T15:16:21,821 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting c341d4df414b4662ac89105376bda5a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731251778901 2024-11-10T15:16:21,822 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting dc290940265f49eaa6aec21344040e34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731251779729 2024-11-10T15:16:21,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/c91e35cd533a4c7c9b531b9ef3a50c16 is 50, key is test_row_0/A:col10/1731251781791/Put/seqid=0 2024-11-10T15:16:21,838 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:16:21,839 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/A is initiating minor compaction (all files) 2024-11-10T15:16:21,839 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/A in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:21,839 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/d4aefe8afc9c4bc78bd47b77ff1c364e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/552b1505c53e4c328428dc23ed868a04, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/472ddaed4e524745a591d7fd79b21d1b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/fe1268fb1bca4b09ab50093ffb20b261] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=49.2 K 2024-11-10T15:16:21,842 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4aefe8afc9c4bc78bd47b77ff1c364e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731251777917 2024-11-10T15:16:21,843 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 552b1505c53e4c328428dc23ed868a04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731251778196 2024-11-10T15:16:21,844 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 472ddaed4e524745a591d7fd79b21d1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731251778901 2024-11-10T15:16:21,845 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe1268fb1bca4b09ab50093ffb20b261, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731251779729 2024-11-10T15:16:21,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741849_1025 (size=14341) 2024-11-10T15:16:21,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/c91e35cd533a4c7c9b531b9ef3a50c16 2024-11-10T15:16:21,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/885d9a0c20494713905f2b3006c69ea0 is 50, key is test_row_0/B:col10/1731251781791/Put/seqid=0 2024-11-10T15:16:21,930 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#A#compaction#13 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:21,941 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/528000699b224ec5ae33bed6bb352562 is 50, key is test_row_0/A:col10/1731251780939/Put/seqid=0 2024-11-10T15:16:21,952 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#B#compaction#15 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:21,953 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/49e66c0ba5364ba7b39bdad59e486794 is 50, key is test_row_0/B:col10/1731251780939/Put/seqid=0 2024-11-10T15:16:21,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741850_1026 (size=12001) 2024-11-10T15:16:22,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741851_1027 (size=12139) 2024-11-10T15:16:22,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741852_1028 (size=12139) 2024-11-10T15:16:22,042 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/528000699b224ec5ae33bed6bb352562 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/528000699b224ec5ae33bed6bb352562 2024-11-10T15:16:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T15:16:22,063 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:22,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:16:22,072 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/49e66c0ba5364ba7b39bdad59e486794 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/49e66c0ba5364ba7b39bdad59e486794 2024-11-10T15:16:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-10T15:16:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T15:16:22,104 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:16:22,113 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:16:22,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:16:22,119 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6bf704d88e72898d090c362728826653/B of 6bf704d88e72898d090c362728826653 into 49e66c0ba5364ba7b39bdad59e486794(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:22,120 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:22,121 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6bf704d88e72898d090c362728826653/A of 6bf704d88e72898d090c362728826653 into 528000699b224ec5ae33bed6bb352562(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:22,121 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:22,124 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/A, priority=12, startTime=1731251781773; duration=0sec 2024-11-10T15:16:22,124 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:22,125 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:A 2024-11-10T15:16:22,125 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:16:22,125 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/B, priority=12, startTime=1731251781795; duration=0sec 2024-11-10T15:16:22,125 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:22,125 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:B 2024-11-10T15:16:22,129 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:16:22,129 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/C is initiating minor compaction (all files) 2024-11-10T15:16:22,129 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/C in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:22,129 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/4bf2a236eaa246bda03b4bce4c11f38a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/9c19487621ad4b95ae57d98cc1af0267, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/856bdef5194e40b7ade42f1330465a5f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/7f0c6acc871949cb8f89aa7355ea82dd] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=46.9 K 2024-11-10T15:16:22,132 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bf2a236eaa246bda03b4bce4c11f38a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731251777917 2024-11-10T15:16:22,134 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c19487621ad4b95ae57d98cc1af0267, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731251778196 2024-11-10T15:16:22,135 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 856bdef5194e40b7ade42f1330465a5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731251778901 2024-11-10T15:16:22,137 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f0c6acc871949cb8f89aa7355ea82dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731251779729 2024-11-10T15:16:22,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251842135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:22,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:22,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251842176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:22,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T15:16:22,208 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#C#compaction#16 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:22,209 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/7f79f0593e504dd8ac1b9f5730f3bd9e is 50, key is test_row_0/C:col10/1731251780939/Put/seqid=0 2024-11-10T15:16:22,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251842243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:22,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251842243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:22,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741853_1029 (size=12139) 2024-11-10T15:16:22,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T15:16:22,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:22,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:22,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:22,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:22,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:22,310 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/7f79f0593e504dd8ac1b9f5730f3bd9e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/7f79f0593e504dd8ac1b9f5730f3bd9e 2024-11-10T15:16:22,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251842282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:22,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251842304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:22,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:22,337 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6bf704d88e72898d090c362728826653/C of 6bf704d88e72898d090c362728826653 into 7f79f0593e504dd8ac1b9f5730f3bd9e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:22,337 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:22,337 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/C, priority=12, startTime=1731251781795; duration=0sec 2024-11-10T15:16:22,338 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:22,338 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:C 2024-11-10T15:16:22,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/885d9a0c20494713905f2b3006c69ea0 2024-11-10T15:16:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T15:16:22,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/65ea4b55e7ee445a99b99a16c854be59 is 50, key is test_row_0/C:col10/1731251781791/Put/seqid=0 2024-11-10T15:16:22,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T15:16:22,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:22,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:22,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:22,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:22,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741854_1030 (size=12001) 2024-11-10T15:16:22,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251842533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:22,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251842545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:22,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:22,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37946 deadline: 1731251842583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:22,599 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_2' in 6bf704d88e72898d090c362728826653 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4389 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:16:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:22,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T15:16:22,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:22,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:22,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:22,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:22,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T15:16:22,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T15:16:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:22,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:22,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251842871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:22,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251842872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:22,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/65ea4b55e7ee445a99b99a16c854be59 2024-11-10T15:16:22,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/c91e35cd533a4c7c9b531b9ef3a50c16 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/c91e35cd533a4c7c9b531b9ef3a50c16 2024-11-10T15:16:22,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T15:16:22,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:22,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:22,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:22,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:22,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:22,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/c91e35cd533a4c7c9b531b9ef3a50c16, entries=200, sequenceid=91, filesize=14.0 K 2024-11-10T15:16:22,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/885d9a0c20494713905f2b3006c69ea0 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/885d9a0c20494713905f2b3006c69ea0 2024-11-10T15:16:23,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/885d9a0c20494713905f2b3006c69ea0, entries=150, sequenceid=91, filesize=11.7 K 2024-11-10T15:16:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/65ea4b55e7ee445a99b99a16c854be59 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/65ea4b55e7ee445a99b99a16c854be59 2024-11-10T15:16:23,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/65ea4b55e7ee445a99b99a16c854be59, entries=150, sequenceid=91, filesize=11.7 K 2024-11-10T15:16:23,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 6bf704d88e72898d090c362728826653 in 1262ms, sequenceid=91, compaction requested=false 2024-11-10T15:16:23,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:23,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T15:16:23,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:23,121 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T15:16:23,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:23,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:23,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:23,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:23,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:23,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:23,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/df8632237f0b499ea5b134ee6e71d6ba is 50, key is test_row_0/A:col10/1731251782168/Put/seqid=0 2024-11-10T15:16:23,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741855_1031 (size=12001) 2024-11-10T15:16:23,179 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/df8632237f0b499ea5b134ee6e71d6ba 2024-11-10T15:16:23,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/28954c2c86e3423ea8bb0c38f282e1fd is 50, key is test_row_0/B:col10/1731251782168/Put/seqid=0 2024-11-10T15:16:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T15:16:23,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741856_1032 (size=12001) 2024-11-10T15:16:23,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:23,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251843371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:23,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:23,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:23,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251843386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:23,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251843402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:23,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:23,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:23,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:23,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251843402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:23,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:23,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:23,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:23,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251843512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:23,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251843512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:23,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:23,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:23,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:23,652 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/28954c2c86e3423ea8bb0c38f282e1fd 2024-11-10T15:16:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/5a33be79665041498f90372283ea7067 is 50, key is test_row_0/C:col10/1731251782168/Put/seqid=0 2024-11-10T15:16:23,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251843722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:23,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741857_1033 (size=12001) 2024-11-10T15:16:23,733 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/5a33be79665041498f90372283ea7067 2024-11-10T15:16:23,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251843732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/df8632237f0b499ea5b134ee6e71d6ba as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/df8632237f0b499ea5b134ee6e71d6ba 2024-11-10T15:16:23,815 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/df8632237f0b499ea5b134ee6e71d6ba, entries=150, sequenceid=114, filesize=11.7 K 2024-11-10T15:16:23,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/28954c2c86e3423ea8bb0c38f282e1fd as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/28954c2c86e3423ea8bb0c38f282e1fd 2024-11-10T15:16:23,837 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/28954c2c86e3423ea8bb0c38f282e1fd, entries=150, sequenceid=114, filesize=11.7 K 2024-11-10T15:16:23,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/5a33be79665041498f90372283ea7067 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/5a33be79665041498f90372283ea7067 2024-11-10T15:16:23,903 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/5a33be79665041498f90372283ea7067, entries=150, sequenceid=114, filesize=11.7 K 2024-11-10T15:16:23,909 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6bf704d88e72898d090c362728826653 in 788ms, sequenceid=114, compaction requested=true 2024-11-10T15:16:23,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:23,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:23,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-10T15:16:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-10T15:16:23,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-10T15:16:23,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8040 sec 2024-11-10T15:16:23,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.8590 sec 2024-11-10T15:16:24,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T15:16:24,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:24,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:24,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:24,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:24,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:24,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:24,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/f9896c464a3e439b940f9be8d524430e is 50, key is test_row_0/A:col10/1731251784034/Put/seqid=0 2024-11-10T15:16:24,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741858_1034 (size=14341) 2024-11-10T15:16:24,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/f9896c464a3e439b940f9be8d524430e 2024-11-10T15:16:24,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/76be9255140a4a258b98661bc3c53e4a is 50, key is test_row_0/B:col10/1731251784034/Put/seqid=0 2024-11-10T15:16:24,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741859_1035 (size=12001) 2024-11-10T15:16:24,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/76be9255140a4a258b98661bc3c53e4a 2024-11-10T15:16:24,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251844156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:24,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:24,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251844156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:24,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:24,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/ebba2cea7bf64c7fafde33af0fd12bdc is 50, key is test_row_0/C:col10/1731251784034/Put/seqid=0 2024-11-10T15:16:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T15:16:24,232 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:24,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:16:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-10T15:16:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T15:16:24,237 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:16:24,239 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:16:24,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:16:24,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741860_1036 (size=12001) 2024-11-10T15:16:24,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/ebba2cea7bf64c7fafde33af0fd12bdc 2024-11-10T15:16:24,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/f9896c464a3e439b940f9be8d524430e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/f9896c464a3e439b940f9be8d524430e 2024-11-10T15:16:24,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/f9896c464a3e439b940f9be8d524430e, entries=200, sequenceid=127, filesize=14.0 K 2024-11-10T15:16:24,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/76be9255140a4a258b98661bc3c53e4a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/76be9255140a4a258b98661bc3c53e4a 2024-11-10T15:16:24,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/76be9255140a4a258b98661bc3c53e4a, entries=150, sequenceid=127, filesize=11.7 K 2024-11-10T15:16:24,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/ebba2cea7bf64c7fafde33af0fd12bdc as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/ebba2cea7bf64c7fafde33af0fd12bdc 2024-11-10T15:16:24,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251844302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:24,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251844312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T15:16:24,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/ebba2cea7bf64c7fafde33af0fd12bdc, entries=150, sequenceid=127, filesize=11.7 K 2024-11-10T15:16:24,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 6bf704d88e72898d090c362728826653 in 312ms, sequenceid=127, compaction requested=true 2024-11-10T15:16:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:16:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:16:24,349 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:16:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:16:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T15:16:24,349 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:16:24,356 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:16:24,356 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/B is initiating minor compaction (all files) 2024-11-10T15:16:24,356 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/B in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:24,356 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52822 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:16:24,357 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/49e66c0ba5364ba7b39bdad59e486794, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/885d9a0c20494713905f2b3006c69ea0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/28954c2c86e3423ea8bb0c38f282e1fd, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/76be9255140a4a258b98661bc3c53e4a] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=47.0 K 2024-11-10T15:16:24,357 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/A is initiating minor compaction (all files) 2024-11-10T15:16:24,357 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/A in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:24,357 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/528000699b224ec5ae33bed6bb352562, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/c91e35cd533a4c7c9b531b9ef3a50c16, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/df8632237f0b499ea5b134ee6e71d6ba, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/f9896c464a3e439b940f9be8d524430e] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=51.6 K 2024-11-10T15:16:24,358 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49e66c0ba5364ba7b39bdad59e486794, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731251779729 2024-11-10T15:16:24,359 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 528000699b224ec5ae33bed6bb352562, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731251779729 2024-11-10T15:16:24,360 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting c91e35cd533a4c7c9b531b9ef3a50c16, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731251781030 2024-11-10T15:16:24,361 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 885d9a0c20494713905f2b3006c69ea0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731251781030 2024-11-10T15:16:24,361 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting df8632237f0b499ea5b134ee6e71d6ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731251781976 2024-11-10T15:16:24,362 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting f9896c464a3e439b940f9be8d524430e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731251783325 2024-11-10T15:16:24,362 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28954c2c86e3423ea8bb0c38f282e1fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731251781976 2024-11-10T15:16:24,363 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76be9255140a4a258b98661bc3c53e4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731251783367 2024-11-10T15:16:24,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T15:16:24,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:24,404 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T15:16:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:24,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:24,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:24,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/324ef17c25114bb4af86b35c5e6291b6 is 50, key is test_row_0/A:col10/1731251784151/Put/seqid=0 2024-11-10T15:16:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:24,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:24,468 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#B#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:24,469 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/0f8b2f9a9a154eef90ebcf87e09dcff4 is 50, key is test_row_0/B:col10/1731251784034/Put/seqid=0 2024-11-10T15:16:24,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741861_1037 (size=12151) 2024-11-10T15:16:24,485 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/324ef17c25114bb4af86b35c5e6291b6 2024-11-10T15:16:24,501 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#A#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:24,502 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/fa550228a3e5428999d64f1a9bb4c63a is 50, key is test_row_0/A:col10/1731251784034/Put/seqid=0 2024-11-10T15:16:24,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741862_1038 (size=12275) 2024-11-10T15:16:24,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:24,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251844524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251844529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T15:16:24,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251844547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:24,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251844547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:24,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/b389e3b5e5014f2bb3a0eb066fae754d is 50, key is test_row_0/B:col10/1731251784151/Put/seqid=0 2024-11-10T15:16:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:24,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741863_1039 (size=12275) 2024-11-10T15:16:24,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741864_1040 (size=12151) 2024-11-10T15:16:24,614 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/fa550228a3e5428999d64f1a9bb4c63a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/fa550228a3e5428999d64f1a9bb4c63a 2024-11-10T15:16:24,632 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6bf704d88e72898d090c362728826653/A of 6bf704d88e72898d090c362728826653 into fa550228a3e5428999d64f1a9bb4c63a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:24,632 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:24,632 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/A, priority=12, startTime=1731251784349; duration=0sec 2024-11-10T15:16:24,633 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:24,633 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:A 2024-11-10T15:16:24,633 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:16:24,635 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:16:24,635 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/C is initiating minor compaction (all files) 2024-11-10T15:16:24,635 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/C in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:24,636 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/7f79f0593e504dd8ac1b9f5730f3bd9e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/65ea4b55e7ee445a99b99a16c854be59, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/5a33be79665041498f90372283ea7067, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/ebba2cea7bf64c7fafde33af0fd12bdc] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=47.0 K 2024-11-10T15:16:24,636 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f79f0593e504dd8ac1b9f5730f3bd9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731251779729 2024-11-10T15:16:24,637 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 65ea4b55e7ee445a99b99a16c854be59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731251781030 2024-11-10T15:16:24,640 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a33be79665041498f90372283ea7067, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731251781976 2024-11-10T15:16:24,652 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting ebba2cea7bf64c7fafde33af0fd12bdc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731251783367 2024-11-10T15:16:24,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251844653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:24,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251844711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:24,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251844720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:24,730 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#C#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:24,731 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/1bfd75b69dcc4c5caa814c0889f54d3c is 50, key is test_row_0/C:col10/1731251784034/Put/seqid=0 2024-11-10T15:16:24,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741865_1041 (size=12275) 2024-11-10T15:16:24,800 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/1bfd75b69dcc4c5caa814c0889f54d3c as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/1bfd75b69dcc4c5caa814c0889f54d3c 2024-11-10T15:16:24,819 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6bf704d88e72898d090c362728826653/C of 6bf704d88e72898d090c362728826653 into 1bfd75b69dcc4c5caa814c0889f54d3c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:24,819 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:24,819 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/C, priority=12, startTime=1731251784349; duration=0sec 2024-11-10T15:16:24,820 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:24,820 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:C 2024-11-10T15:16:24,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251844852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T15:16:24,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251844912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:24,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251844923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:24,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251844932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:24,993 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/0f8b2f9a9a154eef90ebcf87e09dcff4 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/0f8b2f9a9a154eef90ebcf87e09dcff4 2024-11-10T15:16:25,007 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/b389e3b5e5014f2bb3a0eb066fae754d 2024-11-10T15:16:25,009 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6bf704d88e72898d090c362728826653/B of 6bf704d88e72898d090c362728826653 into 0f8b2f9a9a154eef90ebcf87e09dcff4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:25,009 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:25,010 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/B, priority=12, startTime=1731251784349; duration=0sec 2024-11-10T15:16:25,010 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:25,010 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:B 2024-11-10T15:16:25,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/f45f5f900aaf4429ab1300271a0329f2 is 50, key is test_row_0/C:col10/1731251784151/Put/seqid=0 2024-11-10T15:16:25,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741866_1042 (size=12151) 2024-11-10T15:16:25,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:25,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251845222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:25,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:25,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:25,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:25,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251845232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:25,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251845262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:25,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:25,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:25,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:25,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251845362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T15:16:25,462 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/f45f5f900aaf4429ab1300271a0329f2 2024-11-10T15:16:25,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/324ef17c25114bb4af86b35c5e6291b6 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/324ef17c25114bb4af86b35c5e6291b6 2024-11-10T15:16:25,493 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/324ef17c25114bb4af86b35c5e6291b6, entries=150, sequenceid=147, filesize=11.9 K 2024-11-10T15:16:25,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/b389e3b5e5014f2bb3a0eb066fae754d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b389e3b5e5014f2bb3a0eb066fae754d 2024-11-10T15:16:25,507 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b389e3b5e5014f2bb3a0eb066fae754d, entries=150, sequenceid=147, filesize=11.9 K 2024-11-10T15:16:25,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/f45f5f900aaf4429ab1300271a0329f2 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/f45f5f900aaf4429ab1300271a0329f2 2024-11-10T15:16:25,521 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/f45f5f900aaf4429ab1300271a0329f2, entries=150, sequenceid=147, filesize=11.9 K 2024-11-10T15:16:25,540 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 6bf704d88e72898d090c362728826653 in 1136ms, sequenceid=147, compaction requested=false 2024-11-10T15:16:25,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:25,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:25,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-10T15:16:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-10T15:16:25,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-10T15:16:25,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3050 sec 2024-11-10T15:16:25,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3220 sec 2024-11-10T15:16:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:25,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T15:16:25,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:25,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:25,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:25,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:25,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:25,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:25,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/9ae7915489d04bf3b58c2ad5679cd4d8 is 50, key is test_row_0/A:col10/1731251784541/Put/seqid=0 2024-11-10T15:16:25,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741867_1043 (size=12151) 2024-11-10T15:16:25,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/9ae7915489d04bf3b58c2ad5679cd4d8 2024-11-10T15:16:25,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/e7b6689c4c5b45cbb31f008ced456e04 is 50, key is test_row_0/B:col10/1731251784541/Put/seqid=0 2024-11-10T15:16:25,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741868_1044 (size=12151) 2024-11-10T15:16:25,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:25,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251845936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:25,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:25,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:25,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251845936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:25,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251845939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:25,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:25,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:25,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:25,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:25,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:25,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:25,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:25,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:25,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:26,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:26,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251846052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251846052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:26,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251846052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:26,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251846262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:26,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:26,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251846272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:26,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:26,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251846272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:26,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:26,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/e7b6689c4c5b45cbb31f008ced456e04 2024-11-10T15:16:26,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/020b225c65db488e8b074f84398aec93 is 50, key is test_row_0/C:col10/1731251784541/Put/seqid=0 2024-11-10T15:16:26,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741869_1045 (size=12151) 2024-11-10T15:16:26,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251846382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:26,386 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_2' in 6bf704d88e72898d090c362728826653 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:16:26,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T15:16:26,392 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:26,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:16:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-10T15:16:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T15:16:26,396 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:16:26,397 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:16:26,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:16:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T15:16:26,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T15:16:26,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:26,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:26,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:26,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:26,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:26,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251846572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:26,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251846582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:26,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251846582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:26,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:26,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37946 deadline: 1731251846621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:26,625 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_2' in 6bf704d88e72898d090c362728826653 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8415 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:16:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:26,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T15:16:26,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:26,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:26,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:26,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:26,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T15:16:26,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/020b225c65db488e8b074f84398aec93 2024-11-10T15:16:26,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/9ae7915489d04bf3b58c2ad5679cd4d8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/9ae7915489d04bf3b58c2ad5679cd4d8 2024-11-10T15:16:26,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/9ae7915489d04bf3b58c2ad5679cd4d8, entries=150, sequenceid=164, filesize=11.9 K 2024-11-10T15:16:26,764 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-10T15:16:26,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/e7b6689c4c5b45cbb31f008ced456e04 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e7b6689c4c5b45cbb31f008ced456e04 2024-11-10T15:16:26,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e7b6689c4c5b45cbb31f008ced456e04, entries=150, sequenceid=164, filesize=11.9 K 2024-11-10T15:16:26,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/020b225c65db488e8b074f84398aec93 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/020b225c65db488e8b074f84398aec93 2024-11-10T15:16:26,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/020b225c65db488e8b074f84398aec93, entries=150, sequenceid=164, filesize=11.9 K 2024-11-10T15:16:26,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6bf704d88e72898d090c362728826653 in 1045ms, sequenceid=164, compaction requested=true 2024-11-10T15:16:26,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:26,794 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:26,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:16:26,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:26,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:16:26,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:26,795 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:26,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:16:26,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:26,797 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:26,797 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/A is initiating minor compaction (all files) 2024-11-10T15:16:26,797 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/A in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:26,797 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/fa550228a3e5428999d64f1a9bb4c63a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/324ef17c25114bb4af86b35c5e6291b6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/9ae7915489d04bf3b58c2ad5679cd4d8] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=35.7 K 2024-11-10T15:16:26,798 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:26,798 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/B is initiating minor compaction (all files) 2024-11-10T15:16:26,798 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/B in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:26,798 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa550228a3e5428999d64f1a9bb4c63a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731251783367 2024-11-10T15:16:26,798 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/0f8b2f9a9a154eef90ebcf87e09dcff4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b389e3b5e5014f2bb3a0eb066fae754d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e7b6689c4c5b45cbb31f008ced456e04] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=35.7 K 2024-11-10T15:16:26,799 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f8b2f9a9a154eef90ebcf87e09dcff4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731251783367 2024-11-10T15:16:26,799 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 324ef17c25114bb4af86b35c5e6291b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731251784130 2024-11-10T15:16:26,799 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting b389e3b5e5014f2bb3a0eb066fae754d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731251784130 2024-11-10T15:16:26,799 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ae7915489d04bf3b58c2ad5679cd4d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731251784538 2024-11-10T15:16:26,800 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting e7b6689c4c5b45cbb31f008ced456e04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731251784538 2024-11-10T15:16:26,828 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#B#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:26,829 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/af32871db0d246fa9b9a87ef485486af is 50, key is test_row_0/B:col10/1731251784541/Put/seqid=0 2024-11-10T15:16:26,834 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#A#compaction#34 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:26,835 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/f956199a6c7f49ffa8e09d1bf830847f is 50, key is test_row_0/A:col10/1731251784541/Put/seqid=0 2024-11-10T15:16:26,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T15:16:26,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:26,862 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T15:16:26,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:26,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:26,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:26,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:26,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:26,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:26,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741871_1047 (size=12527) 2024-11-10T15:16:26,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/0a5b524d45ef4a97825c3134e507e8fe is 50, key is test_row_0/A:col10/1731251785917/Put/seqid=0 2024-11-10T15:16:26,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741870_1046 (size=12527) 2024-11-10T15:16:26,893 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/af32871db0d246fa9b9a87ef485486af as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/af32871db0d246fa9b9a87ef485486af 2024-11-10T15:16:26,901 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/B of 6bf704d88e72898d090c362728826653 into af32871db0d246fa9b9a87ef485486af(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:26,901 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:26,901 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/B, priority=13, startTime=1731251786795; duration=0sec 2024-11-10T15:16:26,901 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:26,902 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:B 2024-11-10T15:16:26,902 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:26,903 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/f956199a6c7f49ffa8e09d1bf830847f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/f956199a6c7f49ffa8e09d1bf830847f 2024-11-10T15:16:26,903 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:26,904 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/C is initiating minor compaction (all files) 2024-11-10T15:16:26,904 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/C in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:26,904 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/1bfd75b69dcc4c5caa814c0889f54d3c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/f45f5f900aaf4429ab1300271a0329f2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/020b225c65db488e8b074f84398aec93] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=35.7 K 2024-11-10T15:16:26,905 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bfd75b69dcc4c5caa814c0889f54d3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731251783367 2024-11-10T15:16:26,905 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting f45f5f900aaf4429ab1300271a0329f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731251784130 2024-11-10T15:16:26,907 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 020b225c65db488e8b074f84398aec93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731251784538 2024-11-10T15:16:26,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741872_1048 (size=12151) 2024-11-10T15:16:26,934 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/A of 6bf704d88e72898d090c362728826653 into f956199a6c7f49ffa8e09d1bf830847f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:26,934 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:26,934 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/A, priority=13, startTime=1731251786794; duration=0sec 2024-11-10T15:16:26,934 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:26,934 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:A 2024-11-10T15:16:26,943 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#C#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:26,944 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/773f2ab562744a849c51e296eae1d023 is 50, key is test_row_0/C:col10/1731251784541/Put/seqid=0 2024-11-10T15:16:27,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741873_1049 (size=12527) 2024-11-10T15:16:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T15:16:27,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:27,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251847163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:27,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:27,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251847172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:27,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:27,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251847173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:27,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251847281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:27,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:27,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251847291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:27,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251847304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:27,325 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/0a5b524d45ef4a97825c3134e507e8fe 2024-11-10T15:16:27,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/86868782fce44ef2b7f50260083da7e4 is 50, key is test_row_0/B:col10/1731251785917/Put/seqid=0 2024-11-10T15:16:27,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741874_1050 (size=12151) 2024-11-10T15:16:27,358 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/86868782fce44ef2b7f50260083da7e4 2024-11-10T15:16:27,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/79c6b020b2e341168b79844d14246ba3 is 50, key is test_row_0/C:col10/1731251785917/Put/seqid=0 2024-11-10T15:16:27,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741875_1051 (size=12151) 2024-11-10T15:16:27,386 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/79c6b020b2e341168b79844d14246ba3 2024-11-10T15:16:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/0a5b524d45ef4a97825c3134e507e8fe as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/0a5b524d45ef4a97825c3134e507e8fe 2024-11-10T15:16:27,412 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/0a5b524d45ef4a97825c3134e507e8fe, entries=150, sequenceid=182, filesize=11.9 K 2024-11-10T15:16:27,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/86868782fce44ef2b7f50260083da7e4 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/86868782fce44ef2b7f50260083da7e4 2024-11-10T15:16:27,420 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/773f2ab562744a849c51e296eae1d023 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/773f2ab562744a849c51e296eae1d023 2024-11-10T15:16:27,427 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/86868782fce44ef2b7f50260083da7e4, entries=150, sequenceid=182, filesize=11.9 K 2024-11-10T15:16:27,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/79c6b020b2e341168b79844d14246ba3 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/79c6b020b2e341168b79844d14246ba3 2024-11-10T15:16:27,433 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/C of 6bf704d88e72898d090c362728826653 into 773f2ab562744a849c51e296eae1d023(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:27,434 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:27,434 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/C, priority=13, startTime=1731251786795; duration=0sec 2024-11-10T15:16:27,434 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:27,434 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:C 2024-11-10T15:16:27,442 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/79c6b020b2e341168b79844d14246ba3, entries=150, sequenceid=182, filesize=11.9 K 2024-11-10T15:16:27,444 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 6bf704d88e72898d090c362728826653 in 582ms, sequenceid=182, compaction requested=false 2024-11-10T15:16:27,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:27,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:27,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-10T15:16:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-10T15:16:27,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-10T15:16:27,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0490 sec 2024-11-10T15:16:27,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.0590 sec 2024-11-10T15:16:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:27,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T15:16:27,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:27,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:27,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:27,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:27,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:27,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:27,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/ebb189352762453d9d22288010b99256 is 50, key is test_row_0/A:col10/1731251787131/Put/seqid=0 2024-11-10T15:16:27,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741876_1052 (size=14541) 2024-11-10T15:16:27,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/ebb189352762453d9d22288010b99256 2024-11-10T15:16:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T15:16:27,533 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:27,535 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:16:27,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251847531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-10T15:16:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251847537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:27,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T15:16:27,540 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:16:27,541 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:16:27,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:16:27,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251847537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:27,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:27,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/b21ae533c7364fa5a2da7e8dc0ca3219 is 50, key is test_row_0/B:col10/1731251787131/Put/seqid=0 2024-11-10T15:16:27,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741877_1053 (size=12151) 2024-11-10T15:16:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T15:16:27,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251847642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:27,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251847642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:27,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251847651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:27,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T15:16:27,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:27,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:27,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T15:16:27,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:27,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:27,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:27,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:27,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T15:16:27,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251847852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:27,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251847853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:27,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251847862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:27,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/b21ae533c7364fa5a2da7e8dc0ca3219 2024-11-10T15:16:28,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T15:16:28,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:28,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:28,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:28,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:28,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:28,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/a6461a6d5b344a02be7445a7db450530 is 50, key is test_row_0/C:col10/1731251787131/Put/seqid=0 2024-11-10T15:16:28,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741878_1054 (size=12151) 2024-11-10T15:16:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T15:16:28,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T15:16:28,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:28,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:28,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:28,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:28,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:28,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251848162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:28,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251848163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:28,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:28,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251848182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:28,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:28,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T15:16:28,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:28,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:28,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:28,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:28,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:28,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251848402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:28,408 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_2' in 6bf704d88e72898d090c362728826653 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:16:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:28,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T15:16:28,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:28,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:28,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:28,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:28,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:28,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/a6461a6d5b344a02be7445a7db450530 2024-11-10T15:16:28,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/ebb189352762453d9d22288010b99256 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/ebb189352762453d9d22288010b99256 2024-11-10T15:16:28,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/ebb189352762453d9d22288010b99256, entries=200, sequenceid=202, filesize=14.2 K 2024-11-10T15:16:28,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/b21ae533c7364fa5a2da7e8dc0ca3219 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b21ae533c7364fa5a2da7e8dc0ca3219 2024-11-10T15:16:28,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b21ae533c7364fa5a2da7e8dc0ca3219, entries=150, sequenceid=202, filesize=11.9 K 2024-11-10T15:16:28,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/a6461a6d5b344a02be7445a7db450530 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/a6461a6d5b344a02be7445a7db450530 2024-11-10T15:16:28,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/a6461a6d5b344a02be7445a7db450530, entries=150, sequenceid=202, filesize=11.9 K 2024-11-10T15:16:28,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 6bf704d88e72898d090c362728826653 in 1050ms, sequenceid=202, compaction requested=true 2024-11-10T15:16:28,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:28,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:16:28,545 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:28,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:28,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:16:28,546 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:28,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:28,547 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:28,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:16:28,547 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/A is initiating minor compaction (all files) 2024-11-10T15:16:28,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:28,547 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/A in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:28,547 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/f956199a6c7f49ffa8e09d1bf830847f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/0a5b524d45ef4a97825c3134e507e8fe, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/ebb189352762453d9d22288010b99256] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=38.3 K 2024-11-10T15:16:28,548 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:28,548 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/B is initiating minor compaction (all files) 2024-11-10T15:16:28,548 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/B in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:28,548 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/af32871db0d246fa9b9a87ef485486af, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/86868782fce44ef2b7f50260083da7e4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b21ae533c7364fa5a2da7e8dc0ca3219] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=36.0 K 2024-11-10T15:16:28,549 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f956199a6c7f49ffa8e09d1bf830847f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731251784538 2024-11-10T15:16:28,549 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting af32871db0d246fa9b9a87ef485486af, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731251784538 2024-11-10T15:16:28,549 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a5b524d45ef4a97825c3134e507e8fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731251785917 2024-11-10T15:16:28,549 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 86868782fce44ef2b7f50260083da7e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731251785917 2024-11-10T15:16:28,550 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebb189352762453d9d22288010b99256, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731251787131 2024-11-10T15:16:28,550 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting b21ae533c7364fa5a2da7e8dc0ca3219, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731251787131 2024-11-10T15:16:28,567 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#A#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:28,567 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/b77c31983cde4b2cba586a1ba5dae1c0 is 50, key is test_row_0/A:col10/1731251787131/Put/seqid=0 2024-11-10T15:16:28,575 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#B#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:28,576 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/465bd3ae7fc74e3fb63ecb2dbc6bb1e0 is 50, key is test_row_0/B:col10/1731251787131/Put/seqid=0 2024-11-10T15:16:28,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741879_1055 (size=12629) 2024-11-10T15:16:28,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741880_1056 (size=12629) 2024-11-10T15:16:28,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T15:16:28,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:28,633 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T15:16:28,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:28,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:28,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:28,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:28,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:28,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:28,640 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/465bd3ae7fc74e3fb63ecb2dbc6bb1e0 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/465bd3ae7fc74e3fb63ecb2dbc6bb1e0 2024-11-10T15:16:28,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/bde041802cf847ba886e199a59d4418e is 50, key is test_row_0/A:col10/1731251787534/Put/seqid=0 2024-11-10T15:16:28,662 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/B of 6bf704d88e72898d090c362728826653 into 465bd3ae7fc74e3fb63ecb2dbc6bb1e0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:28,662 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:28,662 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/B, priority=13, startTime=1731251788545; duration=0sec 2024-11-10T15:16:28,662 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:28,662 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:B 2024-11-10T15:16:28,662 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:28,664 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:28,664 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/C is initiating minor compaction (all files) 2024-11-10T15:16:28,664 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/C in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:28,665 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/773f2ab562744a849c51e296eae1d023, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/79c6b020b2e341168b79844d14246ba3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/a6461a6d5b344a02be7445a7db450530] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=36.0 K 2024-11-10T15:16:28,665 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 773f2ab562744a849c51e296eae1d023, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731251784538 2024-11-10T15:16:28,666 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 79c6b020b2e341168b79844d14246ba3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731251785917 2024-11-10T15:16:28,666 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting a6461a6d5b344a02be7445a7db450530, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731251787131 2024-11-10T15:16:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T15:16:28,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741881_1057 (size=12151) 2024-11-10T15:16:28,686 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/bde041802cf847ba886e199a59d4418e 2024-11-10T15:16:28,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:28,699 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:28,700 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/310fc32d824a41a9904978be201fbcc2 is 50, key is test_row_0/C:col10/1731251787131/Put/seqid=0 2024-11-10T15:16:28,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/d1d4674096414797931d81c21d74060b is 50, key is test_row_0/B:col10/1731251787534/Put/seqid=0 2024-11-10T15:16:28,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741882_1058 (size=12629) 2024-11-10T15:16:28,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251848727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251848729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:28,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251848736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:28,753 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/310fc32d824a41a9904978be201fbcc2 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/310fc32d824a41a9904978be201fbcc2 2024-11-10T15:16:28,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741883_1059 (size=12151) 2024-11-10T15:16:28,769 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/C of 6bf704d88e72898d090c362728826653 into 310fc32d824a41a9904978be201fbcc2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:28,769 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:28,770 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/C, priority=13, startTime=1731251788547; duration=0sec 2024-11-10T15:16:28,770 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:28,770 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:C 2024-11-10T15:16:28,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251848842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251848843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:28,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251848843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:29,011 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/b77c31983cde4b2cba586a1ba5dae1c0 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/b77c31983cde4b2cba586a1ba5dae1c0 2024-11-10T15:16:29,031 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/A of 6bf704d88e72898d090c362728826653 into b77c31983cde4b2cba586a1ba5dae1c0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:29,031 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:29,031 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/A, priority=13, startTime=1731251788545; duration=0sec 2024-11-10T15:16:29,031 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:29,031 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:A 2024-11-10T15:16:29,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251849051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251849051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:29,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251849061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:29,163 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/d1d4674096414797931d81c21d74060b 2024-11-10T15:16:29,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/77102e17c27946c0afac8842b9edd8c2 is 50, key is test_row_0/C:col10/1731251787534/Put/seqid=0 2024-11-10T15:16:29,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741884_1060 (size=12151) 2024-11-10T15:16:29,220 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/77102e17c27946c0afac8842b9edd8c2 2024-11-10T15:16:29,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/bde041802cf847ba886e199a59d4418e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bde041802cf847ba886e199a59d4418e 2024-11-10T15:16:29,243 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bde041802cf847ba886e199a59d4418e, entries=150, sequenceid=218, filesize=11.9 K 2024-11-10T15:16:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/d1d4674096414797931d81c21d74060b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/d1d4674096414797931d81c21d74060b 2024-11-10T15:16:29,261 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/d1d4674096414797931d81c21d74060b, entries=150, sequenceid=218, filesize=11.9 K 2024-11-10T15:16:29,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/77102e17c27946c0afac8842b9edd8c2 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/77102e17c27946c0afac8842b9edd8c2 2024-11-10T15:16:29,271 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/77102e17c27946c0afac8842b9edd8c2, entries=150, sequenceid=218, filesize=11.9 K 2024-11-10T15:16:29,273 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 6bf704d88e72898d090c362728826653 in 640ms, sequenceid=218, compaction requested=false 2024-11-10T15:16:29,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:29,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:29,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-10T15:16:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-10T15:16:29,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-10T15:16:29,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7350 sec 2024-11-10T15:16:29,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.7480 sec 2024-11-10T15:16:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:29,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T15:16:29,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:29,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:29,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:29,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:29,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:29,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:29,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/1c5e2308d9254eadaf77c9e9421c3a00 is 50, key is test_row_0/A:col10/1731251788732/Put/seqid=0 2024-11-10T15:16:29,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741885_1061 (size=12151) 2024-11-10T15:16:29,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251849464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:29,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251849469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:29,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:29,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251849469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:29,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:29,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251849571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:29,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251849582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:29,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251849582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T15:16:29,682 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:29,683 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:16:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-10T15:16:29,686 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:16:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T15:16:29,687 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:16:29,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:16:29,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251849781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:29,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T15:16:29,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251849792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:29,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251849793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:29,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:29,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:29,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/1c5e2308d9254eadaf77c9e9421c3a00 2024-11-10T15:16:29,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/1435bd41868244de948ce608bd044fb3 is 50, key is test_row_0/B:col10/1731251788732/Put/seqid=0 2024-11-10T15:16:29,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741886_1062 (size=12151) 2024-11-10T15:16:29,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T15:16:29,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:29,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:29,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:29,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:29,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:29,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T15:16:29,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:29,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:29,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:29,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:29,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T15:16:30,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251850092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:30,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:30,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:30,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:30,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251850111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:30,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251850111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:30,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:30,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:30,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:30,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T15:16:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:30,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:30,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/1435bd41868244de948ce608bd044fb3 2024-11-10T15:16:30,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/a6fafc18004f4b0f87e8857ab9c4f9d9 is 50, key is test_row_0/C:col10/1731251788732/Put/seqid=0 2024-11-10T15:16:30,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T15:16:30,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:30,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:30,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:30,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:30,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T15:16:30,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741887_1063 (size=12151) 2024-11-10T15:16:30,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T15:16:30,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:30,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:30,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:30,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:30,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:30,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:30,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T15:16:30,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:30,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:30,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:30,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:30,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:30,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251850622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:30,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251850623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:30,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251850632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:30,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/a6fafc18004f4b0f87e8857ab9c4f9d9 2024-11-10T15:16:30,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/1c5e2308d9254eadaf77c9e9421c3a00 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/1c5e2308d9254eadaf77c9e9421c3a00 2024-11-10T15:16:30,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T15:16:30,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:30,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:30,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:30,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:30,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/1c5e2308d9254eadaf77c9e9421c3a00, entries=150, sequenceid=239, filesize=11.9 K 2024-11-10T15:16:30,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:30,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/1435bd41868244de948ce608bd044fb3 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/1435bd41868244de948ce608bd044fb3 2024-11-10T15:16:30,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/1435bd41868244de948ce608bd044fb3, entries=150, sequenceid=239, filesize=11.9 K 2024-11-10T15:16:30,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/a6fafc18004f4b0f87e8857ab9c4f9d9 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/a6fafc18004f4b0f87e8857ab9c4f9d9 2024-11-10T15:16:30,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/a6fafc18004f4b0f87e8857ab9c4f9d9, entries=150, sequenceid=239, filesize=11.9 K 2024-11-10T15:16:30,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 6bf704d88e72898d090c362728826653 in 1420ms, sequenceid=239, compaction requested=true 2024-11-10T15:16:30,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:30,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:16:30,793 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:30,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:30,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:16:30,793 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:30,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:30,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:16:30,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:30,795 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:30,795 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/B is initiating minor compaction (all files) 2024-11-10T15:16:30,795 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:30,795 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/B in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:30,795 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/A is initiating minor compaction (all files) 2024-11-10T15:16:30,795 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/A in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:30,795 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/465bd3ae7fc74e3fb63ecb2dbc6bb1e0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/d1d4674096414797931d81c21d74060b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/1435bd41868244de948ce608bd044fb3] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=36.1 K 2024-11-10T15:16:30,795 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/b77c31983cde4b2cba586a1ba5dae1c0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bde041802cf847ba886e199a59d4418e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/1c5e2308d9254eadaf77c9e9421c3a00] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=36.1 K 2024-11-10T15:16:30,796 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 465bd3ae7fc74e3fb63ecb2dbc6bb1e0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731251787131 2024-11-10T15:16:30,796 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b77c31983cde4b2cba586a1ba5dae1c0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731251787131 2024-11-10T15:16:30,796 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting d1d4674096414797931d81c21d74060b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731251787521 2024-11-10T15:16:30,796 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting bde041802cf847ba886e199a59d4418e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731251787521 2024-11-10T15:16:30,797 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 1435bd41868244de948ce608bd044fb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731251788726 2024-11-10T15:16:30,797 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c5e2308d9254eadaf77c9e9421c3a00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731251788726 2024-11-10T15:16:30,810 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:30,810 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#A#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:30,811 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/94b7eb21dacf44c5af02cff2189e7249 is 50, key is test_row_0/B:col10/1731251788732/Put/seqid=0 2024-11-10T15:16:30,811 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/762919bf129742e6ac6e41d99c1dd70e is 50, key is test_row_0/A:col10/1731251788732/Put/seqid=0 2024-11-10T15:16:30,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741889_1065 (size=12731) 2024-11-10T15:16:30,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741888_1064 (size=12731) 2024-11-10T15:16:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T15:16:30,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T15:16:30,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:30,928 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T15:16:30,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:30,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:30,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:30,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:30,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:30,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:30,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/888fba5036ef48459e7396ce8af97923 is 50, key is test_row_0/A:col10/1731251789456/Put/seqid=0 2024-11-10T15:16:30,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741890_1066 (size=12151) 2024-11-10T15:16:30,958 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/888fba5036ef48459e7396ce8af97923 2024-11-10T15:16:30,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/b848ecffb68743f5990516cc58a61da8 is 50, key is test_row_0/B:col10/1731251789456/Put/seqid=0 2024-11-10T15:16:30,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741891_1067 (size=12151) 2024-11-10T15:16:30,985 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/b848ecffb68743f5990516cc58a61da8 2024-11-10T15:16:30,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/b3342da04a2e4a6881eb9aaafa0161ac is 50, key is test_row_0/C:col10/1731251789456/Put/seqid=0 2024-11-10T15:16:31,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741892_1068 (size=12151) 2024-11-10T15:16:31,227 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/762919bf129742e6ac6e41d99c1dd70e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/762919bf129742e6ac6e41d99c1dd70e 2024-11-10T15:16:31,227 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/94b7eb21dacf44c5af02cff2189e7249 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/94b7eb21dacf44c5af02cff2189e7249 2024-11-10T15:16:31,235 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/A of 6bf704d88e72898d090c362728826653 into 762919bf129742e6ac6e41d99c1dd70e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:31,236 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:31,236 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/A, priority=13, startTime=1731251790793; duration=0sec 2024-11-10T15:16:31,236 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:31,236 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:A 2024-11-10T15:16:31,236 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:31,237 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/B of 6bf704d88e72898d090c362728826653 into 94b7eb21dacf44c5af02cff2189e7249(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:31,237 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:31,237 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/B, priority=13, startTime=1731251790793; duration=0sec 2024-11-10T15:16:31,237 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:31,237 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:B 2024-11-10T15:16:31,238 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:31,238 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/C is initiating minor compaction (all files) 2024-11-10T15:16:31,238 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/C in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:31,238 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/310fc32d824a41a9904978be201fbcc2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/77102e17c27946c0afac8842b9edd8c2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/a6fafc18004f4b0f87e8857ab9c4f9d9] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=36.1 K 2024-11-10T15:16:31,239 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 310fc32d824a41a9904978be201fbcc2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731251787131 2024-11-10T15:16:31,241 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77102e17c27946c0afac8842b9edd8c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731251787521 2024-11-10T15:16:31,242 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6fafc18004f4b0f87e8857ab9c4f9d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731251788726 2024-11-10T15:16:31,253 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#C#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:31,254 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/59d5f206211b4fd29a3b7304bcc39c29 is 50, key is test_row_0/C:col10/1731251788732/Put/seqid=0 2024-11-10T15:16:31,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741893_1069 (size=12731) 2024-11-10T15:16:31,431 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/b3342da04a2e4a6881eb9aaafa0161ac 2024-11-10T15:16:31,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/888fba5036ef48459e7396ce8af97923 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/888fba5036ef48459e7396ce8af97923 2024-11-10T15:16:31,452 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/888fba5036ef48459e7396ce8af97923, entries=150, sequenceid=254, filesize=11.9 K 2024-11-10T15:16:31,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/b848ecffb68743f5990516cc58a61da8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b848ecffb68743f5990516cc58a61da8 2024-11-10T15:16:31,475 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b848ecffb68743f5990516cc58a61da8, entries=150, sequenceid=254, filesize=11.9 K 2024-11-10T15:16:31,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/b3342da04a2e4a6881eb9aaafa0161ac as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/b3342da04a2e4a6881eb9aaafa0161ac 2024-11-10T15:16:31,488 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/b3342da04a2e4a6881eb9aaafa0161ac, entries=150, sequenceid=254, filesize=11.9 K 2024-11-10T15:16:31,490 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 6bf704d88e72898d090c362728826653 in 561ms, sequenceid=254, compaction requested=false 2024-11-10T15:16:31,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:31,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:31,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-10T15:16:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-10T15:16:31,496 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-10T15:16:31,496 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8050 sec 2024-11-10T15:16:31,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.8140 sec 2024-11-10T15:16:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:31,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T15:16:31,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:31,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:31,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:31,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:31,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:31,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:31,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/e9c223c78d8643b4af9837553db4e011 is 50, key is test_row_0/A:col10/1731251791661/Put/seqid=0 2024-11-10T15:16:31,700 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/59d5f206211b4fd29a3b7304bcc39c29 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/59d5f206211b4fd29a3b7304bcc39c29 2024-11-10T15:16:31,717 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/C of 6bf704d88e72898d090c362728826653 into 59d5f206211b4fd29a3b7304bcc39c29(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:31,717 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:31,717 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/C, priority=13, startTime=1731251790793; duration=0sec 2024-11-10T15:16:31,717 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:31,718 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:C 2024-11-10T15:16:31,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741894_1070 (size=19621) 2024-11-10T15:16:31,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/e9c223c78d8643b4af9837553db4e011 2024-11-10T15:16:31,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/23827a702f3b487eb9af9bd6f7ac7c83 is 50, key is test_row_0/B:col10/1731251791661/Put/seqid=0 2024-11-10T15:16:31,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251851751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:31,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251791702, max timestamp=1731251791753Num uniques 859; 2024-11-10T15:16:31,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:16:31,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251791702, max timestamp=1731251791750Num uniques 859; 2024-11-10T15:16:31,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:16:31,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741895_1071 (size=12301) 2024-11-10T15:16:31,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251851796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:31,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:31,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:31,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:31,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251851799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:31,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:31,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:31,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:31,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T15:16:31,832 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:31,832 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:16:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-10T15:16:31,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T15:16:31,834 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:16:31,835 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:16:31,836 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:16:31,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:31,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251851872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:31,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:31,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:31,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:31,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251851901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:31,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:31,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251851911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:31,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T15:16:31,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T15:16:31,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:31,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:31,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:31,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:31,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:32,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251852081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:32,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:32,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:32,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251852111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:32,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251852122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:32,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:32,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T15:16:32,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:32,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:32,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:32,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:32,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T15:16:32,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/23827a702f3b487eb9af9bd6f7ac7c83 2024-11-10T15:16:32,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/df750749ae0e474891f95598d2ef2205 is 50, key is test_row_0/C:col10/1731251791661/Put/seqid=0 2024-11-10T15:16:32,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741896_1072 (size=12301) 2024-11-10T15:16:32,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T15:16:32,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:32,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:32,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:32,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:32,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:32,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:32,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:32,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251852391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:32,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251852422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:32,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:32,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251852432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:32,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37940 deadline: 1731251852441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:32,443 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_2' in 6bf704d88e72898d090c362728826653 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8288 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:16:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:32,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T15:16:32,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:32,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:32,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:32,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:32,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:32,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T15:16:32,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T15:16:32,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:32,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:32,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:32,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:32,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:32,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/df750749ae0e474891f95598d2ef2205 2024-11-10T15:16:32,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/e9c223c78d8643b4af9837553db4e011 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/e9c223c78d8643b4af9837553db4e011 2024-11-10T15:16:32,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/e9c223c78d8643b4af9837553db4e011, entries=300, sequenceid=267, filesize=19.2 K 2024-11-10T15:16:32,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/23827a702f3b487eb9af9bd6f7ac7c83 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/23827a702f3b487eb9af9bd6f7ac7c83 2024-11-10T15:16:32,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/23827a702f3b487eb9af9bd6f7ac7c83, entries=150, sequenceid=267, filesize=12.0 K 2024-11-10T15:16:32,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/df750749ae0e474891f95598d2ef2205 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/df750749ae0e474891f95598d2ef2205 2024-11-10T15:16:32,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/df750749ae0e474891f95598d2ef2205, entries=150, sequenceid=267, filesize=12.0 K 2024-11-10T15:16:32,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 6bf704d88e72898d090c362728826653 in 987ms, sequenceid=267, compaction requested=true 2024-11-10T15:16:32,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:32,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:16:32,649 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:32,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:32,649 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:32,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:16:32,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:32,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:16:32,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:32,650 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:32,651 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/A is initiating minor compaction (all files) 2024-11-10T15:16:32,651 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/A in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:32,651 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/762919bf129742e6ac6e41d99c1dd70e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/888fba5036ef48459e7396ce8af97923, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/e9c223c78d8643b4af9837553db4e011] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=43.5 K 2024-11-10T15:16:32,651 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 762919bf129742e6ac6e41d99c1dd70e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731251788726 2024-11-10T15:16:32,651 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:32,652 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/B is initiating minor compaction (all files) 2024-11-10T15:16:32,652 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/B in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:32,652 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/94b7eb21dacf44c5af02cff2189e7249, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b848ecffb68743f5990516cc58a61da8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/23827a702f3b487eb9af9bd6f7ac7c83] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=36.3 K 2024-11-10T15:16:32,652 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 94b7eb21dacf44c5af02cff2189e7249, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731251788726 2024-11-10T15:16:32,653 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 888fba5036ef48459e7396ce8af97923, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731251789456 2024-11-10T15:16:32,653 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting b848ecffb68743f5990516cc58a61da8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731251789456 2024-11-10T15:16:32,653 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9c223c78d8643b4af9837553db4e011, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731251791636 2024-11-10T15:16:32,654 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 23827a702f3b487eb9af9bd6f7ac7c83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731251791647 2024-11-10T15:16:32,673 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:32,674 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/8abe2376db4a482aaa5c4b6ee118e410 is 50, key is test_row_0/A:col10/1731251791661/Put/seqid=0 2024-11-10T15:16:32,677 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:32,678 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/236c537ab41d4ae88efbf4f662bd5a73 is 50, key is test_row_0/B:col10/1731251791661/Put/seqid=0 2024-11-10T15:16:32,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741898_1074 (size=12983) 2024-11-10T15:16:32,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741897_1073 (size=12983) 2024-11-10T15:16:32,712 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/8abe2376db4a482aaa5c4b6ee118e410 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/8abe2376db4a482aaa5c4b6ee118e410 2024-11-10T15:16:32,721 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/A of 6bf704d88e72898d090c362728826653 into 8abe2376db4a482aaa5c4b6ee118e410(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:32,721 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:32,721 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/A, priority=13, startTime=1731251792649; duration=0sec 2024-11-10T15:16:32,721 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:32,721 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:A 2024-11-10T15:16:32,722 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:32,723 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:32,723 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/C is initiating minor compaction (all files) 2024-11-10T15:16:32,723 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/C in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:32,723 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/59d5f206211b4fd29a3b7304bcc39c29, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/b3342da04a2e4a6881eb9aaafa0161ac, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/df750749ae0e474891f95598d2ef2205] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=36.3 K 2024-11-10T15:16:32,723 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59d5f206211b4fd29a3b7304bcc39c29, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731251788726 2024-11-10T15:16:32,724 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3342da04a2e4a6881eb9aaafa0161ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731251789456 2024-11-10T15:16:32,724 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting df750749ae0e474891f95598d2ef2205, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731251791647 2024-11-10T15:16:32,734 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#C#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:32,735 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/682a3b5363124f6598618278eb46e66c is 50, key is test_row_0/C:col10/1731251791661/Put/seqid=0 2024-11-10T15:16:32,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741899_1075 (size=12983) 2024-11-10T15:16:32,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T15:16:32,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:32,761 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=125.23 KB heapSize=363.39 KB 2024-11-10T15:16:32,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:32,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:16:32,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:32,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:16:32,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:32,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:16:32,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/3b8333013063418e849e6f80f543e73d is 50, key is test_row_0/A:col10/1731251791794/Put/seqid=0 2024-11-10T15:16:32,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741900_1076 (size=12301) 2024-11-10T15:16:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:32,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:32,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251852934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:32,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251852936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:32,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251852941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:32,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:32,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T15:16:33,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251853042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:33,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251853042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:33,106 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/236c537ab41d4ae88efbf4f662bd5a73 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/236c537ab41d4ae88efbf4f662bd5a73 2024-11-10T15:16:33,114 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/B of 6bf704d88e72898d090c362728826653 into 236c537ab41d4ae88efbf4f662bd5a73(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:33,114 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:33,114 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/B, priority=13, startTime=1731251792649; duration=0sec 2024-11-10T15:16:33,114 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:33,115 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:B 2024-11-10T15:16:33,150 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/682a3b5363124f6598618278eb46e66c as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/682a3b5363124f6598618278eb46e66c 2024-11-10T15:16:33,159 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/C of 6bf704d88e72898d090c362728826653 into 682a3b5363124f6598618278eb46e66c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:33,159 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:33,159 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/C, priority=13, startTime=1731251792650; duration=0sec 2024-11-10T15:16:33,159 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:33,159 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:C 2024-11-10T15:16:33,183 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/3b8333013063418e849e6f80f543e73d 2024-11-10T15:16:33,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/abd5c278f58e4a558aa3a72e3db15870 is 50, key is test_row_0/B:col10/1731251791794/Put/seqid=0 2024-11-10T15:16:33,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741901_1077 (size=12301) 2024-11-10T15:16:33,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:33,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251853252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:33,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:33,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251853252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:33,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:33,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:33,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:33,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:33,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:33,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:33,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251853563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251853563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:33,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:33,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:33,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:33,600 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/abd5c278f58e4a558aa3a72e3db15870 2024-11-10T15:16:33,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/91c0141b5daa43e79954a322a98a7839 is 50, key is test_row_0/C:col10/1731251791794/Put/seqid=0 2024-11-10T15:16:33,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741902_1078 (size=12301) 2024-11-10T15:16:33,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251853951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:33,953 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_1' in 6bf704d88e72898d090c362728826653 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:16:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T15:16:34,032 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/91c0141b5daa43e79954a322a98a7839 2024-11-10T15:16:34,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/3b8333013063418e849e6f80f543e73d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/3b8333013063418e849e6f80f543e73d 2024-11-10T15:16:34,044 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/3b8333013063418e849e6f80f543e73d, entries=150, sequenceid=300, filesize=12.0 K 2024-11-10T15:16:34,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/abd5c278f58e4a558aa3a72e3db15870 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/abd5c278f58e4a558aa3a72e3db15870 2024-11-10T15:16:34,051 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/abd5c278f58e4a558aa3a72e3db15870, entries=150, sequenceid=300, filesize=12.0 K 2024-11-10T15:16:34,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/91c0141b5daa43e79954a322a98a7839 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/91c0141b5daa43e79954a322a98a7839 2024-11-10T15:16:34,059 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/91c0141b5daa43e79954a322a98a7839, entries=150, sequenceid=300, filesize=12.0 K 2024-11-10T15:16:34,060 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~363.34 KB/372064, currentSize=53.67 KB/54960 for 6bf704d88e72898d090c362728826653 in 1299ms, sequenceid=300, compaction requested=false 2024-11-10T15:16:34,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:34,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:34,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-10T15:16:34,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-10T15:16:34,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-10T15:16:34,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2270 sec 2024-11-10T15:16:34,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.2320 sec 2024-11-10T15:16:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:34,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T15:16:34,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:34,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:34,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:34,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:34,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:34,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:34,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/d4f024716a424abfb1f47f72cc115989 is 50, key is test_row_0/A:col10/1731251794074/Put/seqid=0 2024-11-10T15:16:34,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741903_1079 (size=12301) 2024-11-10T15:16:34,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:34,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251854140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:34,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251854147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:34,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/d4f024716a424abfb1f47f72cc115989 2024-11-10T15:16:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:34,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/6aabdbc1809a4a55aa61499d409a0689 is 50, key is test_row_0/B:col10/1731251794074/Put/seqid=0 2024-11-10T15:16:34,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741904_1080 (size=12301) 2024-11-10T15:16:34,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251854252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:34,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251854252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:34,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:34,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251854471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:34,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:34,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251854472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:34,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:34,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:34,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/6aabdbc1809a4a55aa61499d409a0689 2024-11-10T15:16:34,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/30e6539b39f1452f88bb9c9828e34d71 is 50, key is test_row_0/C:col10/1731251794074/Put/seqid=0 2024-11-10T15:16:34,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741905_1081 (size=12301) 2024-11-10T15:16:34,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251854783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:34,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:34,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251854791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:35,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/30e6539b39f1452f88bb9c9828e34d71 2024-11-10T15:16:35,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/d4f024716a424abfb1f47f72cc115989 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/d4f024716a424abfb1f47f72cc115989 2024-11-10T15:16:35,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/d4f024716a424abfb1f47f72cc115989, entries=150, sequenceid=315, filesize=12.0 K 2024-11-10T15:16:35,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/6aabdbc1809a4a55aa61499d409a0689 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/6aabdbc1809a4a55aa61499d409a0689 2024-11-10T15:16:35,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/6aabdbc1809a4a55aa61499d409a0689, entries=150, sequenceid=315, filesize=12.0 K 2024-11-10T15:16:35,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/30e6539b39f1452f88bb9c9828e34d71 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/30e6539b39f1452f88bb9c9828e34d71 2024-11-10T15:16:35,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/30e6539b39f1452f88bb9c9828e34d71, entries=150, sequenceid=315, filesize=12.0 K 2024-11-10T15:16:35,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6bf704d88e72898d090c362728826653 in 994ms, sequenceid=315, compaction requested=true 2024-11-10T15:16:35,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:35,071 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:35,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:16:35,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:35,071 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:35,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:16:35,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:35,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:16:35,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:35,072 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:35,072 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/A is initiating minor compaction (all files) 2024-11-10T15:16:35,072 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/A in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:35,072 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/8abe2376db4a482aaa5c4b6ee118e410, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/3b8333013063418e849e6f80f543e73d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/d4f024716a424abfb1f47f72cc115989] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=36.7 K 2024-11-10T15:16:35,073 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:35,073 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8abe2376db4a482aaa5c4b6ee118e410, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731251791647 2024-11-10T15:16:35,073 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/B is initiating minor compaction (all files) 2024-11-10T15:16:35,073 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/B in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:35,073 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/236c537ab41d4ae88efbf4f662bd5a73, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/abd5c278f58e4a558aa3a72e3db15870, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/6aabdbc1809a4a55aa61499d409a0689] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=36.7 K 2024-11-10T15:16:35,073 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b8333013063418e849e6f80f543e73d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731251791783 2024-11-10T15:16:35,074 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 236c537ab41d4ae88efbf4f662bd5a73, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731251791647 2024-11-10T15:16:35,075 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4f024716a424abfb1f47f72cc115989, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731251792919 2024-11-10T15:16:35,076 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting abd5c278f58e4a558aa3a72e3db15870, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731251791783 2024-11-10T15:16:35,077 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aabdbc1809a4a55aa61499d409a0689, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731251792919 2024-11-10T15:16:35,094 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#B#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:35,095 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/d80da14312444d55bbddcc5f06116e7e is 50, key is test_row_0/B:col10/1731251794074/Put/seqid=0 2024-11-10T15:16:35,115 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#A#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:35,116 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/e15c4d222f724b90bd8e6a540ab14d03 is 50, key is test_row_0/A:col10/1731251794074/Put/seqid=0 2024-11-10T15:16:35,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741906_1082 (size=13085) 2024-11-10T15:16:35,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741907_1083 (size=13085) 2024-11-10T15:16:35,141 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/e15c4d222f724b90bd8e6a540ab14d03 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/e15c4d222f724b90bd8e6a540ab14d03 2024-11-10T15:16:35,156 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/A of 6bf704d88e72898d090c362728826653 into e15c4d222f724b90bd8e6a540ab14d03(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:35,156 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:35,157 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/A, priority=13, startTime=1731251795070; duration=0sec 2024-11-10T15:16:35,160 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:35,160 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:A 2024-11-10T15:16:35,160 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:35,164 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:35,165 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/C is initiating minor compaction (all files) 2024-11-10T15:16:35,165 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/C in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:35,165 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/682a3b5363124f6598618278eb46e66c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/91c0141b5daa43e79954a322a98a7839, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/30e6539b39f1452f88bb9c9828e34d71] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=36.7 K 2024-11-10T15:16:35,166 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 682a3b5363124f6598618278eb46e66c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731251791647 2024-11-10T15:16:35,166 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91c0141b5daa43e79954a322a98a7839, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731251791783 2024-11-10T15:16:35,166 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30e6539b39f1452f88bb9c9828e34d71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731251792919 2024-11-10T15:16:35,187 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#C#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:35,187 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/5056ecf6110b40e19f30c4a8e44529cf is 50, key is test_row_0/C:col10/1731251794074/Put/seqid=0 2024-11-10T15:16:35,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741908_1084 (size=13085) 2024-11-10T15:16:35,217 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/5056ecf6110b40e19f30c4a8e44529cf as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/5056ecf6110b40e19f30c4a8e44529cf 2024-11-10T15:16:35,225 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/C of 6bf704d88e72898d090c362728826653 into 5056ecf6110b40e19f30c4a8e44529cf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:35,225 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:35,225 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/C, priority=13, startTime=1731251795071; duration=0sec 2024-11-10T15:16:35,226 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:35,226 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:C 2024-11-10T15:16:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:35,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T15:16:35,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:35,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:35,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:35,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:35,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:35,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:35,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/9b8d8fc14e2f49cc82eadfb63b496a3d is 50, key is test_row_0/A:col10/1731251795305/Put/seqid=0 2024-11-10T15:16:35,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741909_1085 (size=12301) 2024-11-10T15:16:35,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/9b8d8fc14e2f49cc82eadfb63b496a3d 2024-11-10T15:16:35,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:35,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251855353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:35,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:35,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251855356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:35,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/4f92be33221a41468b0ac19c339e8378 is 50, key is test_row_0/B:col10/1731251795305/Put/seqid=0 2024-11-10T15:16:35,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741910_1086 (size=12301) 2024-11-10T15:16:35,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251855472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:35,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:35,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:35,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:35,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:35,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251855472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:35,542 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/d80da14312444d55bbddcc5f06116e7e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/d80da14312444d55bbddcc5f06116e7e 2024-11-10T15:16:35,557 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/B of 6bf704d88e72898d090c362728826653 into d80da14312444d55bbddcc5f06116e7e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:35,558 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:35,558 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/B, priority=13, startTime=1731251795071; duration=0sec 2024-11-10T15:16:35,558 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:35,558 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:B 2024-11-10T15:16:35,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251855682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:35,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251855691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:35,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/4f92be33221a41468b0ac19c339e8378 2024-11-10T15:16:35,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/eeae037e5de44fe99a524463db3b91fd is 50, key is test_row_0/C:col10/1731251795305/Put/seqid=0 2024-11-10T15:16:35,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741911_1087 (size=12301) 2024-11-10T15:16:35,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/eeae037e5de44fe99a524463db3b91fd 2024-11-10T15:16:35,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/9b8d8fc14e2f49cc82eadfb63b496a3d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/9b8d8fc14e2f49cc82eadfb63b496a3d 2024-11-10T15:16:35,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/9b8d8fc14e2f49cc82eadfb63b496a3d, entries=150, sequenceid=337, filesize=12.0 K 2024-11-10T15:16:35,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/4f92be33221a41468b0ac19c339e8378 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/4f92be33221a41468b0ac19c339e8378 2024-11-10T15:16:35,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/4f92be33221a41468b0ac19c339e8378, entries=150, sequenceid=337, filesize=12.0 K 2024-11-10T15:16:35,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/eeae037e5de44fe99a524463db3b91fd as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/eeae037e5de44fe99a524463db3b91fd 2024-11-10T15:16:35,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/eeae037e5de44fe99a524463db3b91fd, entries=150, sequenceid=337, filesize=12.0 K 2024-11-10T15:16:35,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6bf704d88e72898d090c362728826653 in 564ms, sequenceid=337, compaction requested=false 2024-11-10T15:16:35,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:35,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T15:16:35,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:35,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:35,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:35,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:35,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:35,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:35,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/bdb9a54be15a43eebbdb88ffd8837c23 is 50, key is test_row_0/A:col10/1731251795325/Put/seqid=0 2024-11-10T15:16:35,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741912_1088 (size=14741) 2024-11-10T15:16:35,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/bdb9a54be15a43eebbdb88ffd8837c23 2024-11-10T15:16:35,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/3f98e3b1cb184d599dd683a39523da74 is 50, key is test_row_0/B:col10/1731251795325/Put/seqid=0 2024-11-10T15:16:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T15:16:36,002 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:36,003 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:16:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-10T15:16:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T15:16:36,005 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:16:36,006 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:16:36,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:16:36,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741913_1089 (size=12301) 2024-11-10T15:16:36,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/3f98e3b1cb184d599dd683a39523da74 2024-11-10T15:16:36,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251856022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251856024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251856024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/33aa829fbb4a4892a66e10fb87374097 is 50, key is test_row_0/C:col10/1731251795325/Put/seqid=0 2024-11-10T15:16:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741914_1090 (size=12301) 2024-11-10T15:16:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T15:16:36,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251856141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251856142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251856142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T15:16:36,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:36,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:36,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:36,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:36,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:36,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T15:16:36,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:36,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:36,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:36,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:36,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T15:16:36,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251856352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251856352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251856353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/33aa829fbb4a4892a66e10fb87374097 2024-11-10T15:16:36,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/bdb9a54be15a43eebbdb88ffd8837c23 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bdb9a54be15a43eebbdb88ffd8837c23 2024-11-10T15:16:36,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bdb9a54be15a43eebbdb88ffd8837c23, entries=200, sequenceid=351, filesize=14.4 K 2024-11-10T15:16:36,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/3f98e3b1cb184d599dd683a39523da74 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/3f98e3b1cb184d599dd683a39523da74 2024-11-10T15:16:36,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/3f98e3b1cb184d599dd683a39523da74, entries=150, sequenceid=351, filesize=12.0 K 2024-11-10T15:16:36,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/33aa829fbb4a4892a66e10fb87374097 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/33aa829fbb4a4892a66e10fb87374097 2024-11-10T15:16:36,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T15:16:36,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:36,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:36,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/33aa829fbb4a4892a66e10fb87374097, entries=150, sequenceid=351, filesize=12.0 K 2024-11-10T15:16:36,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6bf704d88e72898d090c362728826653 in 506ms, sequenceid=351, compaction requested=true 2024-11-10T15:16:36,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:36,472 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:36,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:16:36,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:36,472 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:36,473 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:36,474 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/A is initiating minor compaction (all files) 2024-11-10T15:16:36,474 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/A in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:36,474 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/e15c4d222f724b90bd8e6a540ab14d03, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/9b8d8fc14e2f49cc82eadfb63b496a3d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bdb9a54be15a43eebbdb88ffd8837c23] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=39.2 K 2024-11-10T15:16:36,475 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:36,475 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/B is initiating minor compaction (all files) 2024-11-10T15:16:36,475 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/B in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:36,475 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e15c4d222f724b90bd8e6a540ab14d03, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731251792919 2024-11-10T15:16:36,475 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/d80da14312444d55bbddcc5f06116e7e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/4f92be33221a41468b0ac19c339e8378, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/3f98e3b1cb184d599dd683a39523da74] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=36.8 K 2024-11-10T15:16:36,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:16:36,476 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b8d8fc14e2f49cc82eadfb63b496a3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731251794122 2024-11-10T15:16:36,477 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting d80da14312444d55bbddcc5f06116e7e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731251792919 2024-11-10T15:16:36,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:36,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:16:36,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:36,478 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f92be33221a41468b0ac19c339e8378, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731251794122 2024-11-10T15:16:36,479 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f98e3b1cb184d599dd683a39523da74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731251795325 2024-11-10T15:16:36,479 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdb9a54be15a43eebbdb88ffd8837c23, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731251795325 2024-11-10T15:16:36,494 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:36,494 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:36,494 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/804c26ddaf904cbd88356e0a45720d23 is 50, key is test_row_0/B:col10/1731251795325/Put/seqid=0 2024-11-10T15:16:36,494 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/bc38ac0279064ef99de4482d6dd7d779 is 50, key is test_row_0/A:col10/1731251795325/Put/seqid=0 2024-11-10T15:16:36,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741915_1091 (size=13187) 2024-11-10T15:16:36,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741916_1092 (size=13187) 2024-11-10T15:16:36,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T15:16:36,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:36,620 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T15:16:36,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:36,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:36,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:36,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:36,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:36,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:36,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/3f28ca7a4faf4ef19fe0c75692942bf7 is 50, key is test_row_0/A:col10/1731251796022/Put/seqid=0 2024-11-10T15:16:36,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741917_1093 (size=12301) 2024-11-10T15:16:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T15:16:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:36,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. as already flushing 2024-11-10T15:16:36,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251856673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251856672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37946 deadline: 1731251856674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251856683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37946 deadline: 1731251856792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251856791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251856792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251856792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:36,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:36,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:36,910 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/804c26ddaf904cbd88356e0a45720d23 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/804c26ddaf904cbd88356e0a45720d23 2024-11-10T15:16:36,910 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/bc38ac0279064ef99de4482d6dd7d779 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bc38ac0279064ef99de4482d6dd7d779 2024-11-10T15:16:36,917 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/A of 6bf704d88e72898d090c362728826653 into bc38ac0279064ef99de4482d6dd7d779(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:36,918 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:36,918 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/A, priority=13, startTime=1731251796472; duration=0sec 2024-11-10T15:16:36,918 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:36,918 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:A 2024-11-10T15:16:36,918 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:36,919 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/B of 6bf704d88e72898d090c362728826653 into 804c26ddaf904cbd88356e0a45720d23(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:36,919 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:36,919 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/B, priority=13, startTime=1731251796472; duration=0sec 2024-11-10T15:16:36,919 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:36,919 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:B 2024-11-10T15:16:36,919 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:36,919 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/C is initiating minor compaction (all files) 2024-11-10T15:16:36,919 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/C in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:36,920 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/5056ecf6110b40e19f30c4a8e44529cf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/eeae037e5de44fe99a524463db3b91fd, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/33aa829fbb4a4892a66e10fb87374097] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=36.8 K 2024-11-10T15:16:36,922 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5056ecf6110b40e19f30c4a8e44529cf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731251792919 2024-11-10T15:16:36,923 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeae037e5de44fe99a524463db3b91fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731251794122 2024-11-10T15:16:36,924 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33aa829fbb4a4892a66e10fb87374097, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731251795325 2024-11-10T15:16:36,933 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#C#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:36,934 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/2b9701b545814f8fb82ae4a83e308e30 is 50, key is test_row_0/C:col10/1731251795325/Put/seqid=0 2024-11-10T15:16:36,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741918_1094 (size=13187) 2024-11-10T15:16:36,946 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/2b9701b545814f8fb82ae4a83e308e30 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/2b9701b545814f8fb82ae4a83e308e30 2024-11-10T15:16:36,952 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/C of 6bf704d88e72898d090c362728826653 into 2b9701b545814f8fb82ae4a83e308e30(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:36,952 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:36,952 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/C, priority=13, startTime=1731251796478; duration=0sec 2024-11-10T15:16:36,953 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:36,953 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:C 2024-11-10T15:16:37,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:37,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251857002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:37,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:37,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37946 deadline: 1731251857002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251857002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:37,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:37,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251857002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:37,029 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/3f28ca7a4faf4ef19fe0c75692942bf7 2024-11-10T15:16:37,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/e8fc29aa902b4618895aae0d63aeb4b0 is 50, key is test_row_0/B:col10/1731251796022/Put/seqid=0 2024-11-10T15:16:37,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741919_1095 (size=12301) 2024-11-10T15:16:37,054 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/e8fc29aa902b4618895aae0d63aeb4b0 2024-11-10T15:16:37,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/3b9872e96326425f9c9cbcee7d88a30d is 50, key is test_row_0/C:col10/1731251796022/Put/seqid=0 2024-11-10T15:16:37,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741920_1096 (size=12301) 2024-11-10T15:16:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T15:16:37,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251857312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:37,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251857312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:37,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37946 deadline: 1731251857312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:37,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:37,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251857321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:37,479 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/3b9872e96326425f9c9cbcee7d88a30d 2024-11-10T15:16:37,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/3f28ca7a4faf4ef19fe0c75692942bf7 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/3f28ca7a4faf4ef19fe0c75692942bf7 2024-11-10T15:16:37,490 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/3f28ca7a4faf4ef19fe0c75692942bf7, entries=150, sequenceid=370, filesize=12.0 K 2024-11-10T15:16:37,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/e8fc29aa902b4618895aae0d63aeb4b0 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e8fc29aa902b4618895aae0d63aeb4b0 2024-11-10T15:16:37,500 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e8fc29aa902b4618895aae0d63aeb4b0, entries=150, sequenceid=370, filesize=12.0 K 2024-11-10T15:16:37,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/3b9872e96326425f9c9cbcee7d88a30d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/3b9872e96326425f9c9cbcee7d88a30d 2024-11-10T15:16:37,510 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/3b9872e96326425f9c9cbcee7d88a30d, entries=150, sequenceid=370, filesize=12.0 K 2024-11-10T15:16:37,512 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6bf704d88e72898d090c362728826653 in 892ms, sequenceid=370, compaction requested=false 2024-11-10T15:16:37,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:37,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:37,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-10T15:16:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-10T15:16:37,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-10T15:16:37,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5090 sec 2024-11-10T15:16:37,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.5180 sec 2024-11-10T15:16:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:37,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T15:16:37,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:37,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:37,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:37,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:37,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:37,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:37,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/b13c87af06e04db1a0753a69db17f14e is 50, key is test_row_0/A:col10/1731251796672/Put/seqid=0 2024-11-10T15:16:37,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741921_1097 (size=17181) 2024-11-10T15:16:37,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/b13c87af06e04db1a0753a69db17f14e 2024-11-10T15:16:37,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37946 deadline: 1731251857859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:37,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251857861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:37,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:37,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251857862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:37,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251857864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:37,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:37,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:37,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/beecdabd885e48498baad43f71b8b742 is 50, key is test_row_0/B:col10/1731251796672/Put/seqid=0 2024-11-10T15:16:37,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741922_1098 (size=12301) 2024-11-10T15:16:37,905 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-10T15:16:37,905 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:16:37,905 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-10T15:16:37,906 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:16:37,907 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-10T15:16:37,907 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:16:37,912 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:37,912 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:37,912 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:37,912 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:37,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:16:37,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:16:37,913 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:37,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:16:37,913 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:37,918 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-10T15:16:37,918 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:16:37,918 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:37,918 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:37,918 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:16:37,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:37,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:37,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251857972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37946 deadline: 1731251857972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251857972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:37,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251857981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:37,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:37,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T15:16:38,162 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:38,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:38,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:38,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37946 deadline: 1731251858181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251858181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251858181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:38,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251858192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:38,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/beecdabd885e48498baad43f71b8b742 2024-11-10T15:16:38,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/0f4bac2c286041d88f2bc0fd55d9090f is 50, key is test_row_0/C:col10/1731251796672/Put/seqid=0 2024-11-10T15:16:38,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741923_1099 (size=12301) 2024-11-10T15:16:38,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:38,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:38,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37916 deadline: 1731251858492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37972 deadline: 1731251858492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37946 deadline: 1731251858493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:38,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37952 deadline: 1731251858502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:38,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:38,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:16:38,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/0f4bac2c286041d88f2bc0fd55d9090f 2024-11-10T15:16:38,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/b13c87af06e04db1a0753a69db17f14e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/b13c87af06e04db1a0753a69db17f14e 2024-11-10T15:16:38,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/b13c87af06e04db1a0753a69db17f14e, entries=250, sequenceid=388, filesize=16.8 K 2024-11-10T15:16:38,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/beecdabd885e48498baad43f71b8b742 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/beecdabd885e48498baad43f71b8b742 2024-11-10T15:16:38,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/beecdabd885e48498baad43f71b8b742, entries=150, sequenceid=388, filesize=12.0 K 2024-11-10T15:16:38,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/0f4bac2c286041d88f2bc0fd55d9090f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/0f4bac2c286041d88f2bc0fd55d9090f 2024-11-10T15:16:38,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/0f4bac2c286041d88f2bc0fd55d9090f, entries=150, sequenceid=388, filesize=12.0 K 2024-11-10T15:16:38,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6bf704d88e72898d090c362728826653 in 906ms, sequenceid=388, compaction requested=true 2024-11-10T15:16:38,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:38,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:16:38,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:38,740 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:38,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:16:38,740 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:38,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:38,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bf704d88e72898d090c362728826653:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:16:38,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:38,741 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:38,741 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:38,741 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/A is initiating minor compaction (all files) 2024-11-10T15:16:38,741 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/B is initiating minor compaction (all files) 2024-11-10T15:16:38,741 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/B in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:38,741 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/A in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:38,741 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bc38ac0279064ef99de4482d6dd7d779, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/3f28ca7a4faf4ef19fe0c75692942bf7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/b13c87af06e04db1a0753a69db17f14e] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=41.7 K 2024-11-10T15:16:38,741 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/804c26ddaf904cbd88356e0a45720d23, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e8fc29aa902b4618895aae0d63aeb4b0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/beecdabd885e48498baad43f71b8b742] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=36.9 K 2024-11-10T15:16:38,741 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 804c26ddaf904cbd88356e0a45720d23, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731251795325 2024-11-10T15:16:38,741 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc38ac0279064ef99de4482d6dd7d779, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731251795325 2024-11-10T15:16:38,742 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f28ca7a4faf4ef19fe0c75692942bf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731251796014 2024-11-10T15:16:38,742 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting e8fc29aa902b4618895aae0d63aeb4b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731251796014 2024-11-10T15:16:38,742 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b13c87af06e04db1a0753a69db17f14e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731251796666 2024-11-10T15:16:38,742 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting beecdabd885e48498baad43f71b8b742, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731251796666 2024-11-10T15:16:38,767 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#B#compaction#88 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:38,767 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/5cc683acdc6d4070be3a58bceea3dfe9 is 50, key is test_row_0/B:col10/1731251796672/Put/seqid=0 2024-11-10T15:16:38,784 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#A#compaction#87 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:38,785 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/15f01d9fa01740119bda86998550d672 is 50, key is test_row_0/A:col10/1731251796672/Put/seqid=0 2024-11-10T15:16:38,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741924_1100 (size=13289) 2024-11-10T15:16:38,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741925_1101 (size=13289) 2024-11-10T15:16:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 6bf704d88e72898d090c362728826653 2024-11-10T15:16:39,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T15:16:39,004 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-10T15:16:39,004 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:16:39,004 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:39,004 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-10T15:16:39,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:39,004 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:16:39,004 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-10T15:16:39,004 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:39,004 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:39,004 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:16:39,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:39,004 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:39,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:39,004 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:39,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:16:39,004 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:39,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:39,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:16:39,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:39,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:39,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:16:39,014 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-10T15:16:39,014 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:16:39,014 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:39,014 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:39,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:16:39,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/fc8a3d68ef1f415a92f790379004beba is 50, key is test_row_0/A:col10/1731251799002/Put/seqid=0 2024-11-10T15:16:39,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741926_1102 (size=12301) 2024-11-10T15:16:39,196 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/5cc683acdc6d4070be3a58bceea3dfe9 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/5cc683acdc6d4070be3a58bceea3dfe9 2024-11-10T15:16:39,202 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/15f01d9fa01740119bda86998550d672 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/15f01d9fa01740119bda86998550d672 2024-11-10T15:16:39,202 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/B of 6bf704d88e72898d090c362728826653 into 5cc683acdc6d4070be3a58bceea3dfe9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:39,202 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:39,202 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/B, priority=13, startTime=1731251798740; duration=0sec 2024-11-10T15:16:39,203 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:39,203 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:B 2024-11-10T15:16:39,203 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:39,204 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:39,204 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 6bf704d88e72898d090c362728826653/C is initiating minor compaction (all files) 2024-11-10T15:16:39,204 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bf704d88e72898d090c362728826653/C in TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:39,204 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/2b9701b545814f8fb82ae4a83e308e30, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/3b9872e96326425f9c9cbcee7d88a30d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/0f4bac2c286041d88f2bc0fd55d9090f] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp, totalSize=36.9 K 2024-11-10T15:16:39,205 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b9701b545814f8fb82ae4a83e308e30, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731251795325 2024-11-10T15:16:39,205 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b9872e96326425f9c9cbcee7d88a30d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731251796014 2024-11-10T15:16:39,206 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f4bac2c286041d88f2bc0fd55d9090f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731251796666 2024-11-10T15:16:39,209 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/A of 6bf704d88e72898d090c362728826653 into 15f01d9fa01740119bda86998550d672(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:39,209 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:39,209 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/A, priority=13, startTime=1731251798740; duration=0sec 2024-11-10T15:16:39,209 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:39,209 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:A 2024-11-10T15:16:39,218 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bf704d88e72898d090c362728826653#C#compaction#90 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:39,219 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/131b34c6245741a998a1fc5197024053 is 50, key is test_row_0/C:col10/1731251796672/Put/seqid=0 2024-11-10T15:16:39,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741927_1103 (size=13289) 2024-11-10T15:16:39,425 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T15:16:39,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/fc8a3d68ef1f415a92f790379004beba 2024-11-10T15:16:39,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/7dd8ef7280d44bacb009dfb29e6fd220 is 50, key is test_row_0/B:col10/1731251799002/Put/seqid=0 2024-11-10T15:16:39,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741928_1104 (size=12301) 2024-11-10T15:16:39,638 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/131b34c6245741a998a1fc5197024053 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/131b34c6245741a998a1fc5197024053 2024-11-10T15:16:39,644 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bf704d88e72898d090c362728826653/C of 6bf704d88e72898d090c362728826653 into 131b34c6245741a998a1fc5197024053(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:39,644 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:39,644 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., storeName=6bf704d88e72898d090c362728826653/C, priority=13, startTime=1731251798740; duration=0sec 2024-11-10T15:16:39,644 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:39,644 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bf704d88e72898d090c362728826653:C 2024-11-10T15:16:39,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/7dd8ef7280d44bacb009dfb29e6fd220 2024-11-10T15:16:39,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/9f0200d118f24490a2c2bf4aacd59065 is 50, key is test_row_0/C:col10/1731251799002/Put/seqid=0 2024-11-10T15:16:39,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741929_1105 (size=12301) 2024-11-10T15:16:40,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/9f0200d118f24490a2c2bf4aacd59065 2024-11-10T15:16:40,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/fc8a3d68ef1f415a92f790379004beba as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/fc8a3d68ef1f415a92f790379004beba 2024-11-10T15:16:40,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/fc8a3d68ef1f415a92f790379004beba, entries=150, sequenceid=408, filesize=12.0 K 2024-11-10T15:16:40,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/7dd8ef7280d44bacb009dfb29e6fd220 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/7dd8ef7280d44bacb009dfb29e6fd220 2024-11-10T15:16:40,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/7dd8ef7280d44bacb009dfb29e6fd220, entries=150, sequenceid=408, filesize=12.0 K 2024-11-10T15:16:40,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/9f0200d118f24490a2c2bf4aacd59065 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/9f0200d118f24490a2c2bf4aacd59065 2024-11-10T15:16:40,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/9f0200d118f24490a2c2bf4aacd59065, entries=150, sequenceid=408, filesize=12.0 K 2024-11-10T15:16:40,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=6.71 KB/6870 for 6bf704d88e72898d090c362728826653 in 1283ms, sequenceid=408, compaction requested=false 2024-11-10T15:16:40,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bf704d88e72898d090c362728826653: 2024-11-10T15:16:42,504 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-10T15:16:42,504 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:16:42,504 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:42,504 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:42,504 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:16:42,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T15:16:42,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-10T15:16:42,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-10T15:16:42,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-10T15:16:42,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-10T15:16:42,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-10T15:16:42,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T15:16:42,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1986 2024-11-10T15:16:42,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1922 2024-11-10T15:16:42,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T15:16:42,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 896 2024-11-10T15:16:42,506 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2683 rows 2024-11-10T15:16:42,506 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 893 2024-11-10T15:16:42,506 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2677 rows 2024-11-10T15:16:42,506 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T15:16:42,506 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T15:16:42,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:42,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:42,506 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:16:42,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T15:16:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T15:16:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T15:16:42,519 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251802519"}]},"ts":"1731251802519"} 2024-11-10T15:16:42,522 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T15:16:42,522 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T15:16:42,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T15:16:42,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bf704d88e72898d090c362728826653, UNASSIGN}] 2024-11-10T15:16:42,531 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bf704d88e72898d090c362728826653, UNASSIGN 2024-11-10T15:16:42,532 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=6bf704d88e72898d090c362728826653, regionState=CLOSING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:16:42,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bf704d88e72898d090c362728826653, UNASSIGN because future has completed 2024-11-10T15:16:42,535 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T15:16:42,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:16:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T15:16:42,692 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 6bf704d88e72898d090c362728826653 2024-11-10T15:16:42,692 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T15:16:42,693 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 6bf704d88e72898d090c362728826653, disabling compactions & flushes 2024-11-10T15:16:42,693 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:42,693 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:42,693 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. after waiting 0 ms 2024-11-10T15:16:42,693 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:42,693 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 6bf704d88e72898d090c362728826653 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-10T15:16:42,694 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=A 2024-11-10T15:16:42,694 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:42,694 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=B 2024-11-10T15:16:42,694 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:42,694 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bf704d88e72898d090c362728826653, store=C 2024-11-10T15:16:42,694 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:42,698 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/9a8b0e68ccf1465b93596e36a3873235 is 50, key is test_row_1/A:col10/1731251799012/Put/seqid=0 2024-11-10T15:16:42,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741930_1106 (size=9857) 2024-11-10T15:16:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T15:16:43,106 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/9a8b0e68ccf1465b93596e36a3873235 2024-11-10T15:16:43,116 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/240a26e7055446e3b8245043254e058c is 50, key is test_row_1/B:col10/1731251799012/Put/seqid=0 2024-11-10T15:16:43,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741931_1107 (size=9857) 2024-11-10T15:16:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T15:16:43,523 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/240a26e7055446e3b8245043254e058c 2024-11-10T15:16:43,537 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/4fea96561e2640b6b3e7754c585eb0fc is 50, key is test_row_1/C:col10/1731251799012/Put/seqid=0 2024-11-10T15:16:43,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741932_1108 (size=9857) 2024-11-10T15:16:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T15:16:43,943 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/4fea96561e2640b6b3e7754c585eb0fc 2024-11-10T15:16:43,953 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/A/9a8b0e68ccf1465b93596e36a3873235 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/9a8b0e68ccf1465b93596e36a3873235 2024-11-10T15:16:43,971 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/9a8b0e68ccf1465b93596e36a3873235, entries=100, sequenceid=416, filesize=9.6 K 2024-11-10T15:16:43,972 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/B/240a26e7055446e3b8245043254e058c as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/240a26e7055446e3b8245043254e058c 2024-11-10T15:16:43,981 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/240a26e7055446e3b8245043254e058c, entries=100, sequenceid=416, filesize=9.6 K 2024-11-10T15:16:43,983 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/.tmp/C/4fea96561e2640b6b3e7754c585eb0fc as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/4fea96561e2640b6b3e7754c585eb0fc 2024-11-10T15:16:43,994 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/4fea96561e2640b6b3e7754c585eb0fc, entries=100, sequenceid=416, filesize=9.6 K 2024-11-10T15:16:43,996 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 6bf704d88e72898d090c362728826653 in 1303ms, sequenceid=416, compaction requested=true 2024-11-10T15:16:44,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/d4aefe8afc9c4bc78bd47b77ff1c364e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/552b1505c53e4c328428dc23ed868a04, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/472ddaed4e524745a591d7fd79b21d1b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/fe1268fb1bca4b09ab50093ffb20b261, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/528000699b224ec5ae33bed6bb352562, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/c91e35cd533a4c7c9b531b9ef3a50c16, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/df8632237f0b499ea5b134ee6e71d6ba, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/f9896c464a3e439b940f9be8d524430e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/fa550228a3e5428999d64f1a9bb4c63a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/324ef17c25114bb4af86b35c5e6291b6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/f956199a6c7f49ffa8e09d1bf830847f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/9ae7915489d04bf3b58c2ad5679cd4d8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/0a5b524d45ef4a97825c3134e507e8fe, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/ebb189352762453d9d22288010b99256, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/b77c31983cde4b2cba586a1ba5dae1c0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bde041802cf847ba886e199a59d4418e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/762919bf129742e6ac6e41d99c1dd70e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/1c5e2308d9254eadaf77c9e9421c3a00, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/888fba5036ef48459e7396ce8af97923, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/e9c223c78d8643b4af9837553db4e011, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/8abe2376db4a482aaa5c4b6ee118e410, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/3b8333013063418e849e6f80f543e73d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/e15c4d222f724b90bd8e6a540ab14d03, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/d4f024716a424abfb1f47f72cc115989, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/9b8d8fc14e2f49cc82eadfb63b496a3d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bdb9a54be15a43eebbdb88ffd8837c23, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bc38ac0279064ef99de4482d6dd7d779, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/3f28ca7a4faf4ef19fe0c75692942bf7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/b13c87af06e04db1a0753a69db17f14e] to archive 2024-11-10T15:16:44,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:16:44,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/d4aefe8afc9c4bc78bd47b77ff1c364e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/d4aefe8afc9c4bc78bd47b77ff1c364e 2024-11-10T15:16:44,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/552b1505c53e4c328428dc23ed868a04 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/552b1505c53e4c328428dc23ed868a04 2024-11-10T15:16:44,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/472ddaed4e524745a591d7fd79b21d1b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/472ddaed4e524745a591d7fd79b21d1b 2024-11-10T15:16:44,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/fe1268fb1bca4b09ab50093ffb20b261 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/fe1268fb1bca4b09ab50093ffb20b261 2024-11-10T15:16:44,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/528000699b224ec5ae33bed6bb352562 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/528000699b224ec5ae33bed6bb352562 2024-11-10T15:16:44,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/c91e35cd533a4c7c9b531b9ef3a50c16 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/c91e35cd533a4c7c9b531b9ef3a50c16 2024-11-10T15:16:44,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/df8632237f0b499ea5b134ee6e71d6ba to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/df8632237f0b499ea5b134ee6e71d6ba 2024-11-10T15:16:44,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/f9896c464a3e439b940f9be8d524430e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/f9896c464a3e439b940f9be8d524430e 2024-11-10T15:16:44,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/fa550228a3e5428999d64f1a9bb4c63a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/fa550228a3e5428999d64f1a9bb4c63a 2024-11-10T15:16:44,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/324ef17c25114bb4af86b35c5e6291b6 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/324ef17c25114bb4af86b35c5e6291b6 2024-11-10T15:16:44,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/f956199a6c7f49ffa8e09d1bf830847f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/f956199a6c7f49ffa8e09d1bf830847f 2024-11-10T15:16:44,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/9ae7915489d04bf3b58c2ad5679cd4d8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/9ae7915489d04bf3b58c2ad5679cd4d8 2024-11-10T15:16:44,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/0a5b524d45ef4a97825c3134e507e8fe to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/0a5b524d45ef4a97825c3134e507e8fe 2024-11-10T15:16:44,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/ebb189352762453d9d22288010b99256 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/ebb189352762453d9d22288010b99256 2024-11-10T15:16:44,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/b77c31983cde4b2cba586a1ba5dae1c0 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/b77c31983cde4b2cba586a1ba5dae1c0 2024-11-10T15:16:44,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bde041802cf847ba886e199a59d4418e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bde041802cf847ba886e199a59d4418e 2024-11-10T15:16:44,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/762919bf129742e6ac6e41d99c1dd70e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/762919bf129742e6ac6e41d99c1dd70e 2024-11-10T15:16:44,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/1c5e2308d9254eadaf77c9e9421c3a00 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/1c5e2308d9254eadaf77c9e9421c3a00 2024-11-10T15:16:44,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/888fba5036ef48459e7396ce8af97923 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/888fba5036ef48459e7396ce8af97923 2024-11-10T15:16:44,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/e9c223c78d8643b4af9837553db4e011 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/e9c223c78d8643b4af9837553db4e011 2024-11-10T15:16:44,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/8abe2376db4a482aaa5c4b6ee118e410 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/8abe2376db4a482aaa5c4b6ee118e410 2024-11-10T15:16:44,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/3b8333013063418e849e6f80f543e73d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/3b8333013063418e849e6f80f543e73d 2024-11-10T15:16:44,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/e15c4d222f724b90bd8e6a540ab14d03 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/e15c4d222f724b90bd8e6a540ab14d03 2024-11-10T15:16:44,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/d4f024716a424abfb1f47f72cc115989 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/d4f024716a424abfb1f47f72cc115989 2024-11-10T15:16:44,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/9b8d8fc14e2f49cc82eadfb63b496a3d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/9b8d8fc14e2f49cc82eadfb63b496a3d 2024-11-10T15:16:44,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bdb9a54be15a43eebbdb88ffd8837c23 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bdb9a54be15a43eebbdb88ffd8837c23 2024-11-10T15:16:44,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bc38ac0279064ef99de4482d6dd7d779 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/bc38ac0279064ef99de4482d6dd7d779 2024-11-10T15:16:44,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/3f28ca7a4faf4ef19fe0c75692942bf7 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/3f28ca7a4faf4ef19fe0c75692942bf7 2024-11-10T15:16:44,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/b13c87af06e04db1a0753a69db17f14e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/b13c87af06e04db1a0753a69db17f14e 2024-11-10T15:16:44,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e6e7563283c44bb39c338fc031bd8fc1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/05fbd66649854448a4ef4bbd7015cbe8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/c341d4df414b4662ac89105376bda5a9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/49e66c0ba5364ba7b39bdad59e486794, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/dc290940265f49eaa6aec21344040e34, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/885d9a0c20494713905f2b3006c69ea0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/28954c2c86e3423ea8bb0c38f282e1fd, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/0f8b2f9a9a154eef90ebcf87e09dcff4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/76be9255140a4a258b98661bc3c53e4a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b389e3b5e5014f2bb3a0eb066fae754d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/af32871db0d246fa9b9a87ef485486af, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e7b6689c4c5b45cbb31f008ced456e04, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/86868782fce44ef2b7f50260083da7e4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/465bd3ae7fc74e3fb63ecb2dbc6bb1e0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b21ae533c7364fa5a2da7e8dc0ca3219, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/d1d4674096414797931d81c21d74060b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/94b7eb21dacf44c5af02cff2189e7249, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/1435bd41868244de948ce608bd044fb3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b848ecffb68743f5990516cc58a61da8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/236c537ab41d4ae88efbf4f662bd5a73, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/23827a702f3b487eb9af9bd6f7ac7c83, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/abd5c278f58e4a558aa3a72e3db15870, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/d80da14312444d55bbddcc5f06116e7e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/6aabdbc1809a4a55aa61499d409a0689, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/4f92be33221a41468b0ac19c339e8378, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/804c26ddaf904cbd88356e0a45720d23, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/3f98e3b1cb184d599dd683a39523da74, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e8fc29aa902b4618895aae0d63aeb4b0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/beecdabd885e48498baad43f71b8b742] to archive 2024-11-10T15:16:44,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:16:44,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e6e7563283c44bb39c338fc031bd8fc1 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e6e7563283c44bb39c338fc031bd8fc1 2024-11-10T15:16:44,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/05fbd66649854448a4ef4bbd7015cbe8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/05fbd66649854448a4ef4bbd7015cbe8 2024-11-10T15:16:44,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/c341d4df414b4662ac89105376bda5a9 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/c341d4df414b4662ac89105376bda5a9 2024-11-10T15:16:44,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/49e66c0ba5364ba7b39bdad59e486794 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/49e66c0ba5364ba7b39bdad59e486794 2024-11-10T15:16:44,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/dc290940265f49eaa6aec21344040e34 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/dc290940265f49eaa6aec21344040e34 2024-11-10T15:16:44,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/885d9a0c20494713905f2b3006c69ea0 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/885d9a0c20494713905f2b3006c69ea0 2024-11-10T15:16:44,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/28954c2c86e3423ea8bb0c38f282e1fd to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/28954c2c86e3423ea8bb0c38f282e1fd 2024-11-10T15:16:44,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/0f8b2f9a9a154eef90ebcf87e09dcff4 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/0f8b2f9a9a154eef90ebcf87e09dcff4 2024-11-10T15:16:44,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/76be9255140a4a258b98661bc3c53e4a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/76be9255140a4a258b98661bc3c53e4a 2024-11-10T15:16:44,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b389e3b5e5014f2bb3a0eb066fae754d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b389e3b5e5014f2bb3a0eb066fae754d 2024-11-10T15:16:44,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/af32871db0d246fa9b9a87ef485486af to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/af32871db0d246fa9b9a87ef485486af 2024-11-10T15:16:44,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e7b6689c4c5b45cbb31f008ced456e04 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e7b6689c4c5b45cbb31f008ced456e04 2024-11-10T15:16:44,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/86868782fce44ef2b7f50260083da7e4 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/86868782fce44ef2b7f50260083da7e4 2024-11-10T15:16:44,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/465bd3ae7fc74e3fb63ecb2dbc6bb1e0 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/465bd3ae7fc74e3fb63ecb2dbc6bb1e0 2024-11-10T15:16:44,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b21ae533c7364fa5a2da7e8dc0ca3219 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b21ae533c7364fa5a2da7e8dc0ca3219 2024-11-10T15:16:44,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/d1d4674096414797931d81c21d74060b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/d1d4674096414797931d81c21d74060b 2024-11-10T15:16:44,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/94b7eb21dacf44c5af02cff2189e7249 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/94b7eb21dacf44c5af02cff2189e7249 2024-11-10T15:16:44,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/1435bd41868244de948ce608bd044fb3 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/1435bd41868244de948ce608bd044fb3 2024-11-10T15:16:44,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b848ecffb68743f5990516cc58a61da8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/b848ecffb68743f5990516cc58a61da8 2024-11-10T15:16:44,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/236c537ab41d4ae88efbf4f662bd5a73 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/236c537ab41d4ae88efbf4f662bd5a73 2024-11-10T15:16:44,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/23827a702f3b487eb9af9bd6f7ac7c83 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/23827a702f3b487eb9af9bd6f7ac7c83 2024-11-10T15:16:44,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/abd5c278f58e4a558aa3a72e3db15870 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/abd5c278f58e4a558aa3a72e3db15870 2024-11-10T15:16:44,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/d80da14312444d55bbddcc5f06116e7e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/d80da14312444d55bbddcc5f06116e7e 2024-11-10T15:16:44,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/6aabdbc1809a4a55aa61499d409a0689 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/6aabdbc1809a4a55aa61499d409a0689 2024-11-10T15:16:44,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/4f92be33221a41468b0ac19c339e8378 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/4f92be33221a41468b0ac19c339e8378 2024-11-10T15:16:44,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/804c26ddaf904cbd88356e0a45720d23 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/804c26ddaf904cbd88356e0a45720d23 2024-11-10T15:16:44,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/3f98e3b1cb184d599dd683a39523da74 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/3f98e3b1cb184d599dd683a39523da74 2024-11-10T15:16:44,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e8fc29aa902b4618895aae0d63aeb4b0 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/e8fc29aa902b4618895aae0d63aeb4b0 2024-11-10T15:16:44,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/beecdabd885e48498baad43f71b8b742 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/beecdabd885e48498baad43f71b8b742 2024-11-10T15:16:44,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/4bf2a236eaa246bda03b4bce4c11f38a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/9c19487621ad4b95ae57d98cc1af0267, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/856bdef5194e40b7ade42f1330465a5f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/7f79f0593e504dd8ac1b9f5730f3bd9e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/7f0c6acc871949cb8f89aa7355ea82dd, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/65ea4b55e7ee445a99b99a16c854be59, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/5a33be79665041498f90372283ea7067, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/1bfd75b69dcc4c5caa814c0889f54d3c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/ebba2cea7bf64c7fafde33af0fd12bdc, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/f45f5f900aaf4429ab1300271a0329f2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/773f2ab562744a849c51e296eae1d023, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/020b225c65db488e8b074f84398aec93, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/79c6b020b2e341168b79844d14246ba3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/310fc32d824a41a9904978be201fbcc2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/a6461a6d5b344a02be7445a7db450530, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/77102e17c27946c0afac8842b9edd8c2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/59d5f206211b4fd29a3b7304bcc39c29, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/a6fafc18004f4b0f87e8857ab9c4f9d9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/b3342da04a2e4a6881eb9aaafa0161ac, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/682a3b5363124f6598618278eb46e66c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/df750749ae0e474891f95598d2ef2205, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/91c0141b5daa43e79954a322a98a7839, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/5056ecf6110b40e19f30c4a8e44529cf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/30e6539b39f1452f88bb9c9828e34d71, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/eeae037e5de44fe99a524463db3b91fd, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/2b9701b545814f8fb82ae4a83e308e30, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/33aa829fbb4a4892a66e10fb87374097, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/3b9872e96326425f9c9cbcee7d88a30d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/0f4bac2c286041d88f2bc0fd55d9090f] to archive 2024-11-10T15:16:44,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:16:44,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/4bf2a236eaa246bda03b4bce4c11f38a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/4bf2a236eaa246bda03b4bce4c11f38a 2024-11-10T15:16:44,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/9c19487621ad4b95ae57d98cc1af0267 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/9c19487621ad4b95ae57d98cc1af0267 2024-11-10T15:16:44,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/856bdef5194e40b7ade42f1330465a5f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/856bdef5194e40b7ade42f1330465a5f 2024-11-10T15:16:44,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/7f79f0593e504dd8ac1b9f5730f3bd9e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/7f79f0593e504dd8ac1b9f5730f3bd9e 2024-11-10T15:16:44,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/7f0c6acc871949cb8f89aa7355ea82dd to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/7f0c6acc871949cb8f89aa7355ea82dd 2024-11-10T15:16:44,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/65ea4b55e7ee445a99b99a16c854be59 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/65ea4b55e7ee445a99b99a16c854be59 2024-11-10T15:16:44,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/5a33be79665041498f90372283ea7067 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/5a33be79665041498f90372283ea7067 2024-11-10T15:16:44,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/1bfd75b69dcc4c5caa814c0889f54d3c to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/1bfd75b69dcc4c5caa814c0889f54d3c 2024-11-10T15:16:44,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/ebba2cea7bf64c7fafde33af0fd12bdc to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/ebba2cea7bf64c7fafde33af0fd12bdc 2024-11-10T15:16:44,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/f45f5f900aaf4429ab1300271a0329f2 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/f45f5f900aaf4429ab1300271a0329f2 2024-11-10T15:16:44,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/773f2ab562744a849c51e296eae1d023 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/773f2ab562744a849c51e296eae1d023 2024-11-10T15:16:44,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/020b225c65db488e8b074f84398aec93 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/020b225c65db488e8b074f84398aec93 2024-11-10T15:16:44,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/79c6b020b2e341168b79844d14246ba3 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/79c6b020b2e341168b79844d14246ba3 2024-11-10T15:16:44,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/310fc32d824a41a9904978be201fbcc2 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/310fc32d824a41a9904978be201fbcc2 2024-11-10T15:16:44,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/a6461a6d5b344a02be7445a7db450530 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/a6461a6d5b344a02be7445a7db450530 2024-11-10T15:16:44,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/77102e17c27946c0afac8842b9edd8c2 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/77102e17c27946c0afac8842b9edd8c2 2024-11-10T15:16:44,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/59d5f206211b4fd29a3b7304bcc39c29 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/59d5f206211b4fd29a3b7304bcc39c29 2024-11-10T15:16:44,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/a6fafc18004f4b0f87e8857ab9c4f9d9 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/a6fafc18004f4b0f87e8857ab9c4f9d9 2024-11-10T15:16:44,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/b3342da04a2e4a6881eb9aaafa0161ac to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/b3342da04a2e4a6881eb9aaafa0161ac 2024-11-10T15:16:44,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/682a3b5363124f6598618278eb46e66c to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/682a3b5363124f6598618278eb46e66c 2024-11-10T15:16:44,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/df750749ae0e474891f95598d2ef2205 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/df750749ae0e474891f95598d2ef2205 2024-11-10T15:16:44,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/91c0141b5daa43e79954a322a98a7839 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/91c0141b5daa43e79954a322a98a7839 2024-11-10T15:16:44,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/5056ecf6110b40e19f30c4a8e44529cf to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/5056ecf6110b40e19f30c4a8e44529cf 2024-11-10T15:16:44,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/30e6539b39f1452f88bb9c9828e34d71 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/30e6539b39f1452f88bb9c9828e34d71 2024-11-10T15:16:44,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/eeae037e5de44fe99a524463db3b91fd to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/eeae037e5de44fe99a524463db3b91fd 2024-11-10T15:16:44,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/2b9701b545814f8fb82ae4a83e308e30 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/2b9701b545814f8fb82ae4a83e308e30 2024-11-10T15:16:44,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/33aa829fbb4a4892a66e10fb87374097 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/33aa829fbb4a4892a66e10fb87374097 2024-11-10T15:16:44,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/3b9872e96326425f9c9cbcee7d88a30d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/3b9872e96326425f9c9cbcee7d88a30d 2024-11-10T15:16:44,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/0f4bac2c286041d88f2bc0fd55d9090f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/0f4bac2c286041d88f2bc0fd55d9090f 2024-11-10T15:16:44,327 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/recovered.edits/419.seqid, newMaxSeqId=419, maxSeqId=1 2024-11-10T15:16:44,328 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. 2024-11-10T15:16:44,329 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 6bf704d88e72898d090c362728826653: Waiting for close lock at 1731251802693Running coprocessor pre-close hooks at 1731251802693Disabling compacts and flushes for region at 1731251802693Disabling writes for close at 1731251802693Obtaining lock to block concurrent updates at 1731251802694 (+1 ms)Preparing flush snapshotting stores in 6bf704d88e72898d090c362728826653 at 1731251802694Finished memstore snapshotting TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731251802694Flushing stores of TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653. at 1731251802695 (+1 ms)Flushing 6bf704d88e72898d090c362728826653/A: creating writer at 1731251802695Flushing 6bf704d88e72898d090c362728826653/A: appending metadata at 1731251802698 (+3 ms)Flushing 6bf704d88e72898d090c362728826653/A: closing flushed file at 1731251802698Flushing 6bf704d88e72898d090c362728826653/B: creating writer at 1731251803111 (+413 ms)Flushing 6bf704d88e72898d090c362728826653/B: appending metadata at 1731251803115 (+4 ms)Flushing 6bf704d88e72898d090c362728826653/B: closing flushed file at 1731251803116 (+1 ms)Flushing 6bf704d88e72898d090c362728826653/C: creating writer at 1731251803532 (+416 ms)Flushing 6bf704d88e72898d090c362728826653/C: appending metadata at 1731251803536 (+4 ms)Flushing 6bf704d88e72898d090c362728826653/C: closing flushed file at 1731251803536Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5078a5f9: reopening flushed file at 1731251803951 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b0875a2: reopening flushed file at 1731251803971 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c1f7359: reopening flushed file at 1731251803981 (+10 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 6bf704d88e72898d090c362728826653 in 1303ms, sequenceid=416, compaction requested=true at 1731251803996 (+15 ms)Writing region close event to WAL at 1731251804316 (+320 ms)Running coprocessor post-close hooks at 1731251804328 (+12 ms)Closed at 1731251804328 2024-11-10T15:16:44,335 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=6bf704d88e72898d090c362728826653, regionState=CLOSED 2024-11-10T15:16:44,335 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 6bf704d88e72898d090c362728826653 2024-11-10T15:16:44,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:16:44,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-10T15:16:44,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 6bf704d88e72898d090c362728826653, server=c0771061be61,46087,1731251771284 in 1.8060 sec 2024-11-10T15:16:44,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-10T15:16:44,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bf704d88e72898d090c362728826653, UNASSIGN in 1.8160 sec 2024-11-10T15:16:44,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-10T15:16:44,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8270 sec 2024-11-10T15:16:44,358 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251804358"}]},"ts":"1731251804358"} 2024-11-10T15:16:44,361 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T15:16:44,361 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T15:16:44,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8510 sec 2024-11-10T15:16:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T15:16:44,673 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:44,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T15:16:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:16:44,689 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:16:44,692 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:16:44,697 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653 2024-11-10T15:16:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T15:16:44,702 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/recovered.edits] 2024-11-10T15:16:44,709 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/15f01d9fa01740119bda86998550d672 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/15f01d9fa01740119bda86998550d672 2024-11-10T15:16:44,711 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/9a8b0e68ccf1465b93596e36a3873235 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/9a8b0e68ccf1465b93596e36a3873235 2024-11-10T15:16:44,714 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/fc8a3d68ef1f415a92f790379004beba to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/A/fc8a3d68ef1f415a92f790379004beba 2024-11-10T15:16:44,718 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/240a26e7055446e3b8245043254e058c to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/240a26e7055446e3b8245043254e058c 2024-11-10T15:16:44,721 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/5cc683acdc6d4070be3a58bceea3dfe9 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/5cc683acdc6d4070be3a58bceea3dfe9 2024-11-10T15:16:44,732 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/7dd8ef7280d44bacb009dfb29e6fd220 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/B/7dd8ef7280d44bacb009dfb29e6fd220 2024-11-10T15:16:44,738 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/131b34c6245741a998a1fc5197024053 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/131b34c6245741a998a1fc5197024053 2024-11-10T15:16:44,741 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/4fea96561e2640b6b3e7754c585eb0fc to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/4fea96561e2640b6b3e7754c585eb0fc 2024-11-10T15:16:44,743 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/9f0200d118f24490a2c2bf4aacd59065 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/C/9f0200d118f24490a2c2bf4aacd59065 2024-11-10T15:16:44,750 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/recovered.edits/419.seqid to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653/recovered.edits/419.seqid 2024-11-10T15:16:44,751 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/6bf704d88e72898d090c362728826653 2024-11-10T15:16:44,751 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T15:16:44,759 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:16:44,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-10T15:16:44,771 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T15:16:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T15:16:44,836 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T15:16:44,838 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:16:44,839 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T15:16:44,839 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731251804839"}]},"ts":"9223372036854775807"} 2024-11-10T15:16:44,846 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T15:16:44,846 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6bf704d88e72898d090c362728826653, NAME => 'TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T15:16:44,847 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T15:16:44,847 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731251804847"}]},"ts":"9223372036854775807"} 2024-11-10T15:16:44,851 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T15:16:44,852 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:16:44,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 174 msec 2024-11-10T15:16:45,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T15:16:45,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T15:16:45,013 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:45,029 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=246 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/c0771061be61:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;c0771061be61:46087-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/c0771061be61:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2087031749_22 at /127.0.0.1:34524 [Waiting for operation #224] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1405295892_22 at /127.0.0.1:45460 [Waiting for operation #235] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=454 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=773 (was 585) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3692 (was 4239) 2024-11-10T15:16:45,040 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=246, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=773, ProcessCount=11, AvailableMemoryMB=3691 2024-11-10T15:16:45,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T15:16:45,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T15:16:45,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T15:16:45,045 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T15:16:45,045 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:45,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-10T15:16:45,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-10T15:16:45,047 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T15:16:45,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741933_1109 (size=1061) 2024-11-10T15:16:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-10T15:16:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-10T15:16:45,492 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85 2024-11-10T15:16:45,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741934_1110 (size=53) 2024-11-10T15:16:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-10T15:16:45,903 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:16:45,903 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7c1cb26053f9cb07691f27c558e31c12, disabling compactions & flushes 2024-11-10T15:16:45,903 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:45,903 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:45,903 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. after waiting 0 ms 2024-11-10T15:16:45,903 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:45,903 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:45,903 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7c1cb26053f9cb07691f27c558e31c12: Waiting for close lock at 1731251805903Disabling compacts and flushes for region at 1731251805903Disabling writes for close at 1731251805903Writing region close event to WAL at 1731251805903Closed at 1731251805903 2024-11-10T15:16:45,904 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T15:16:45,905 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731251805905"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731251805905"}]},"ts":"1731251805905"} 2024-11-10T15:16:45,907 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T15:16:45,908 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T15:16:45,908 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251805908"}]},"ts":"1731251805908"} 2024-11-10T15:16:45,910 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T15:16:45,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c1cb26053f9cb07691f27c558e31c12, ASSIGN}] 2024-11-10T15:16:45,911 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c1cb26053f9cb07691f27c558e31c12, ASSIGN 2024-11-10T15:16:45,912 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c1cb26053f9cb07691f27c558e31c12, ASSIGN; state=OFFLINE, location=c0771061be61,46087,1731251771284; forceNewPlan=false, retain=false 2024-11-10T15:16:46,063 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=7c1cb26053f9cb07691f27c558e31c12, regionState=OPENING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:16:46,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c1cb26053f9cb07691f27c558e31c12, ASSIGN because future has completed 2024-11-10T15:16:46,066 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:16:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-10T15:16:46,222 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:46,222 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} 2024-11-10T15:16:46,222 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:46,223 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:16:46,223 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:46,223 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:46,225 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:46,226 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:16:46,227 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c1cb26053f9cb07691f27c558e31c12 columnFamilyName A 2024-11-10T15:16:46,228 DEBUG [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:46,228 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.HStore(327): Store=7c1cb26053f9cb07691f27c558e31c12/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:16:46,229 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:46,230 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:16:46,231 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c1cb26053f9cb07691f27c558e31c12 columnFamilyName B 2024-11-10T15:16:46,231 DEBUG [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:46,232 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.HStore(327): Store=7c1cb26053f9cb07691f27c558e31c12/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:16:46,232 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:46,233 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:16:46,234 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c1cb26053f9cb07691f27c558e31c12 columnFamilyName C 2024-11-10T15:16:46,234 DEBUG [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:46,234 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.HStore(327): Store=7c1cb26053f9cb07691f27c558e31c12/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:16:46,235 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:46,235 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:46,235 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:46,236 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:46,237 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:46,237 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:46,237 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T15:16:46,238 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:46,240 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T15:16:46,241 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 7c1cb26053f9cb07691f27c558e31c12; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64148223, jitterRate=-0.04411698877811432}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T15:16:46,241 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:46,242 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 7c1cb26053f9cb07691f27c558e31c12: Running coprocessor pre-open hook at 1731251806223Writing region info on filesystem at 1731251806223Initializing all the Stores at 1731251806224 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251806224Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251806225 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251806225Cleaning up temporary data from old regions at 1731251806237 (+12 ms)Running coprocessor post-open hooks at 1731251806241 (+4 ms)Region opened successfully at 1731251806242 (+1 ms) 2024-11-10T15:16:46,243 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., pid=30, masterSystemTime=1731251806218 2024-11-10T15:16:46,245 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:46,245 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:46,245 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=7c1cb26053f9cb07691f27c558e31c12, regionState=OPEN, openSeqNum=2, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:16:46,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:16:46,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-10T15:16:46,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 in 182 msec 2024-11-10T15:16:46,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-10T15:16:46,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c1cb26053f9cb07691f27c558e31c12, ASSIGN in 340 msec 2024-11-10T15:16:46,253 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T15:16:46,254 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251806253"}]},"ts":"1731251806253"} 2024-11-10T15:16:46,256 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T15:16:46,257 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T15:16:46,259 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2150 sec 2024-11-10T15:16:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-10T15:16:47,202 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:47,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:47,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:47,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:47,204 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@150e67fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:47,206 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:47,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cedc1ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:47,207 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:47,209 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:47,211 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T15:16:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is c0771061be61,41501,1731251770419 2024-11-10T15:16:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17eb0dc9 2024-11-10T15:16:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T15:16:47,213 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T15:16:47,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T15:16:47,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T15:16:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T15:16:47,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741935_1111 (size=1097) 2024-11-10T15:16:47,636 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T15:16:47,637 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T15:16:47,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T15:16:47,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c1cb26053f9cb07691f27c558e31c12, REOPEN/MOVE}] 2024-11-10T15:16:47,647 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c1cb26053f9cb07691f27c558e31c12, REOPEN/MOVE 2024-11-10T15:16:47,648 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=7c1cb26053f9cb07691f27c558e31c12, regionState=CLOSING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:16:47,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c1cb26053f9cb07691f27c558e31c12, REOPEN/MOVE because future has completed 2024-11-10T15:16:47,650 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T15:16:47,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:16:47,804 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:47,804 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T15:16:47,804 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 7c1cb26053f9cb07691f27c558e31c12, disabling compactions & flushes 2024-11-10T15:16:47,804 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:47,804 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:47,804 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. after waiting 0 ms 2024-11-10T15:16:47,804 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:47,811 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T15:16:47,813 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:47,813 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 7c1cb26053f9cb07691f27c558e31c12: Waiting for close lock at 1731251807804Running coprocessor pre-close hooks at 1731251807804Disabling compacts and flushes for region at 1731251807804Disabling writes for close at 1731251807804Writing region close event to WAL at 1731251807805 (+1 ms)Running coprocessor post-close hooks at 1731251807813 (+8 ms)Closed at 1731251807813 2024-11-10T15:16:47,813 WARN [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 7c1cb26053f9cb07691f27c558e31c12 to self. 2024-11-10T15:16:47,817 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:47,817 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=7c1cb26053f9cb07691f27c558e31c12, regionState=CLOSED 2024-11-10T15:16:47,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:16:47,821 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-10T15:16:47,822 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 in 170 msec 2024-11-10T15:16:47,822 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c1cb26053f9cb07691f27c558e31c12, REOPEN/MOVE; state=CLOSED, location=c0771061be61,46087,1731251771284; forceNewPlan=false, retain=true 2024-11-10T15:16:47,973 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=7c1cb26053f9cb07691f27c558e31c12, regionState=OPENING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:16:47,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c1cb26053f9cb07691f27c558e31c12, REOPEN/MOVE because future has completed 2024-11-10T15:16:47,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:16:48,134 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:48,134 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} 2024-11-10T15:16:48,134 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,134 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:16:48,134 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,135 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,137 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,137 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:16:48,141 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c1cb26053f9cb07691f27c558e31c12 columnFamilyName A 2024-11-10T15:16:48,143 DEBUG [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:48,145 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.HStore(327): Store=7c1cb26053f9cb07691f27c558e31c12/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:16:48,145 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,146 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:16:48,146 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c1cb26053f9cb07691f27c558e31c12 columnFamilyName B 2024-11-10T15:16:48,147 DEBUG [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:48,147 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.HStore(327): Store=7c1cb26053f9cb07691f27c558e31c12/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:16:48,147 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,148 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:16:48,148 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c1cb26053f9cb07691f27c558e31c12 columnFamilyName C 2024-11-10T15:16:48,148 DEBUG [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:48,149 INFO [StoreOpener-7c1cb26053f9cb07691f27c558e31c12-1 {}] regionserver.HStore(327): Store=7c1cb26053f9cb07691f27c558e31c12/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:16:48,149 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:48,149 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,150 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,151 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,152 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,152 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,152 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T15:16:48,154 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,154 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 7c1cb26053f9cb07691f27c558e31c12; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75334472, jitterRate=0.12257111072540283}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T15:16:48,154 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,155 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 7c1cb26053f9cb07691f27c558e31c12: Running coprocessor pre-open hook at 1731251808135Writing region info on filesystem at 1731251808135Initializing all the Stores at 1731251808135Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251808135Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251808136 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251808136Cleaning up temporary data from old regions at 1731251808152 (+16 ms)Running coprocessor post-open hooks at 1731251808155 (+3 ms)Region opened successfully at 1731251808155 2024-11-10T15:16:48,156 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., pid=35, masterSystemTime=1731251808131 2024-11-10T15:16:48,158 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:48,158 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:48,158 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=7c1cb26053f9cb07691f27c558e31c12, regionState=OPEN, openSeqNum=5, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:16:48,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-10T15:16:48,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 in 184 msec 2024-11-10T15:16:48,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-10T15:16:48,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c1cb26053f9cb07691f27c558e31c12, REOPEN/MOVE in 516 msec 2024-11-10T15:16:48,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-10T15:16:48,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 525 msec 2024-11-10T15:16:48,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 949 msec 2024-11-10T15:16:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-10T15:16:48,173 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:48,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,174 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:48,174 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:48,175 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@356c8867, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:48,177 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:48,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d1d783, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,179 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:48,179 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:48,180 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:48,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:48,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:48,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411e8aaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:48,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:48,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:48,181 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bf9cec6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:48,183 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:48,184 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26cb47c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:48,186 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:48,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29018818, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:48,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:48,188 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:48,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:48,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:48,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59891bb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:48,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:48,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:48,190 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eab7741, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,191 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:48,191 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:48,192 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:48,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36c30e0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:48,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:48,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:48,194 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46492bf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:48,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:48,197 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d7f49fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:48,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:48,198 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b3eb06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:48,200 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:48,201 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dfc90db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:48,203 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7203a827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:48,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:48,205 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bfb8bf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:48,207 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:48,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7825eb66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,208 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:16:48,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:16:48,209 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:16:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:16:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:16:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f574288, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:16:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:16:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:16:48,210 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:16:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cfa90e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:16:48,212 DEBUG [Thread-590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:48,212 DEBUG [Thread-577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:48,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:48,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:48,213 DEBUG [Thread-581 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:48,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:48,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:48,214 DEBUG [Thread-584 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:48,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:48,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:48,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:48,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:48,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:48,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:48,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:48,216 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:16:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-10T15:16:48,218 DEBUG [Thread-575 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T15:16:48,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:48,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:48,219 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:16:48,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:48,220 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:16:48,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:16:48,220 DEBUG [Thread-588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:16:48,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:48,222 DEBUG [Thread-588 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:48,222 DEBUG [Thread-588 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:48,222 DEBUG [Thread-579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:48,223 DEBUG [Thread-573 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:16:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:16:48,224 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:48,225 DEBUG [Thread-586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:16:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:16:48,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:48,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:48,228 DEBUG [Thread-586 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:16:48,229 DEBUG [Thread-586 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:16:48,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:16:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:16:48,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:16:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:16:48,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:16:48,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:16:48,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T15:16:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:16:48,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:48,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:16:48,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:48,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:16:48,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:48,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d8f4accc65b040dfae9584aea9584b71_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251808243/Put/seqid=0 2024-11-10T15:16:48,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251808289, max timestamp=1731251808305Num uniques 859; 2024-11-10T15:16:48,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:16:48,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251808289, max timestamp=1731251808305Num uniques 859; 2024-11-10T15:16:48,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:16:48,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251808289, max timestamp=1731251808304Num uniques 859; 2024-11-10T15:16:48,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:16:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T15:16:48,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741936_1112 (size=14594) 2024-11-10T15:16:48,323 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:48,331 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d8f4accc65b040dfae9584aea9584b71_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d8f4accc65b040dfae9584aea9584b71_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,334 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/8d3eba0879ea422ca53958c3a520acc4, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:48,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/8d3eba0879ea422ca53958c3a520acc4 is 175, key is test_row_0/A:col10/1731251808243/Put/seqid=0 2024-11-10T15:16:48,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251868341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251868352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251868354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251868358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251868359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741937_1113 (size=39549) 2024-11-10T15:16:48,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T15:16:48,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:48,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:48,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:48,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:48,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:48,386 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/8d3eba0879ea422ca53958c3a520acc4 2024-11-10T15:16:48,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/618e9e669b8945b689fbdd18c48ae688 is 50, key is test_row_0/B:col10/1731251808243/Put/seqid=0 2024-11-10T15:16:48,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741938_1114 (size=12001) 2024-11-10T15:16:48,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/618e9e669b8945b689fbdd18c48ae688 2024-11-10T15:16:48,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251868469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251868473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251868473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/6b68ae09e5c64517abd50edaeac6a598 is 50, key is test_row_0/C:col10/1731251808243/Put/seqid=0 2024-11-10T15:16:48,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251868473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251868473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741939_1115 (size=12001) 2024-11-10T15:16:48,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/6b68ae09e5c64517abd50edaeac6a598 2024-11-10T15:16:48,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/8d3eba0879ea422ca53958c3a520acc4 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/8d3eba0879ea422ca53958c3a520acc4 2024-11-10T15:16:48,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/8d3eba0879ea422ca53958c3a520acc4, entries=200, sequenceid=15, filesize=38.6 K 2024-11-10T15:16:48,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/618e9e669b8945b689fbdd18c48ae688 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/618e9e669b8945b689fbdd18c48ae688 2024-11-10T15:16:48,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/618e9e669b8945b689fbdd18c48ae688, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T15:16:48,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/6b68ae09e5c64517abd50edaeac6a598 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/6b68ae09e5c64517abd50edaeac6a598 2024-11-10T15:16:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T15:16:48,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/6b68ae09e5c64517abd50edaeac6a598, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T15:16:48,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 7c1cb26053f9cb07691f27c558e31c12 in 290ms, sequenceid=15, compaction requested=false 2024-11-10T15:16:48,534 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T15:16:48,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:48,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T15:16:48,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:48,539 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-10T15:16:48,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:16:48,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:16:48,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:16:48,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:16:48,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:16:48,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:16:48,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111019aa6e77513e460daf2514e7f0a691f3_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251808354/Put/seqid=0 2024-11-10T15:16:48,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741940_1116 (size=12154) 2024-11-10T15:16:48,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:48,594 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111019aa6e77513e460daf2514e7f0a691f3_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111019aa6e77513e460daf2514e7f0a691f3_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/edfa00bd356043078b1d8fdb3e415837, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:48,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/edfa00bd356043078b1d8fdb3e415837 is 175, key is test_row_0/A:col10/1731251808354/Put/seqid=0 2024-11-10T15:16:48,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741941_1117 (size=30955) 2024-11-10T15:16:48,617 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/edfa00bd356043078b1d8fdb3e415837 2024-11-10T15:16:48,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/e0f12cc24bc845518f2f23b1ca64a623 is 50, key is test_row_0/B:col10/1731251808354/Put/seqid=0 2024-11-10T15:16:48,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741942_1118 (size=12001) 2024-11-10T15:16:48,640 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/e0f12cc24bc845518f2f23b1ca64a623 2024-11-10T15:16:48,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d3ebe27c94674e05a05ba4ae1757b0a2 is 50, key is test_row_0/C:col10/1731251808354/Put/seqid=0 2024-11-10T15:16:48,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741943_1119 (size=12001) 2024-11-10T15:16:48,674 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d3ebe27c94674e05a05ba4ae1757b0a2 2024-11-10T15:16:48,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/edfa00bd356043078b1d8fdb3e415837 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/edfa00bd356043078b1d8fdb3e415837 2024-11-10T15:16:48,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,695 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/edfa00bd356043078b1d8fdb3e415837, entries=150, sequenceid=50, filesize=30.2 K 2024-11-10T15:16:48,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-10T15:16:48,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/e0f12cc24bc845518f2f23b1ca64a623 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e0f12cc24bc845518f2f23b1ca64a623 2024-11-10T15:16:48,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251868697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251868699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251868699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,708 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e0f12cc24bc845518f2f23b1ca64a623, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T15:16:48,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d3ebe27c94674e05a05ba4ae1757b0a2 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d3ebe27c94674e05a05ba4ae1757b0a2 2024-11-10T15:16:48,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251868701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,716 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d3ebe27c94674e05a05ba4ae1757b0a2, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T15:16:48,718 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 7c1cb26053f9cb07691f27c558e31c12 in 180ms, sequenceid=50, compaction requested=false 2024-11-10T15:16:48,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:48,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:48,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-10T15:16:48,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-10T15:16:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T15:16:48,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:16:48,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:48,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:16:48,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:48,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:16:48,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:48,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-10T15:16:48,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 501 msec 2024-11-10T15:16:48,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 510 msec 2024-11-10T15:16:48,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107b85a342ec634515adfc9b873ff0d671_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251808719/Put/seqid=0 2024-11-10T15:16:48,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741944_1120 (size=19474) 2024-11-10T15:16:48,778 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:48,785 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107b85a342ec634515adfc9b873ff0d671_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107b85a342ec634515adfc9b873ff0d671_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:48,790 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/4196dc98e4694744be8e34a53b037c1c, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:48,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/4196dc98e4694744be8e34a53b037c1c is 175, key is test_row_0/A:col10/1731251808719/Put/seqid=0 2024-11-10T15:16:48,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741945_1121 (size=56733) 2024-11-10T15:16:48,805 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/4196dc98e4694744be8e34a53b037c1c 2024-11-10T15:16:48,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/584209bbb9a2470e8211a0461daf0e18 is 50, key is test_row_0/B:col10/1731251808719/Put/seqid=0 2024-11-10T15:16:48,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741946_1122 (size=12001) 2024-11-10T15:16:48,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/584209bbb9a2470e8211a0461daf0e18 2024-11-10T15:16:48,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T15:16:48,842 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:48,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251868823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,842 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:16:48,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251868824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-10T15:16:48,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251868826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251868828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251868831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T15:16:48,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,848 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:16:48,849 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:16:48,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:16:48,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/23ac4329a03e44b98c48bd8bd4a28ed4 is 50, key is test_row_0/C:col10/1731251808719/Put/seqid=0 2024-11-10T15:16:48,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741947_1123 (size=12001) 2024-11-10T15:16:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T15:16:48,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251868952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251868952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251868952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251868952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:48,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251868953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T15:16:49,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:49,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:49,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:49,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:49,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:49,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T15:16:49,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:49,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:49,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:49,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:49,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:49,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T15:16:49,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:49,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251869162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251869162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251869162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251869163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251869163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/23ac4329a03e44b98c48bd8bd4a28ed4 2024-11-10T15:16:49,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T15:16:49,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:49,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:49,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:49,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:49,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/4196dc98e4694744be8e34a53b037c1c as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/4196dc98e4694744be8e34a53b037c1c 2024-11-10T15:16:49,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:49,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:49,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/4196dc98e4694744be8e34a53b037c1c, entries=300, sequenceid=62, filesize=55.4 K 2024-11-10T15:16:49,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/584209bbb9a2470e8211a0461daf0e18 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/584209bbb9a2470e8211a0461daf0e18 2024-11-10T15:16:49,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/584209bbb9a2470e8211a0461daf0e18, entries=150, sequenceid=62, filesize=11.7 K 2024-11-10T15:16:49,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/23ac4329a03e44b98c48bd8bd4a28ed4 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/23ac4329a03e44b98c48bd8bd4a28ed4 2024-11-10T15:16:49,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/23ac4329a03e44b98c48bd8bd4a28ed4, entries=150, sequenceid=62, filesize=11.7 K 2024-11-10T15:16:49,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7c1cb26053f9cb07691f27c558e31c12 in 658ms, sequenceid=62, compaction requested=true 2024-11-10T15:16:49,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:49,379 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:49,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:16:49,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:49,380 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:49,381 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:49,381 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/A is initiating minor compaction (all files) 2024-11-10T15:16:49,381 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/A in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:49,381 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/8d3eba0879ea422ca53958c3a520acc4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/edfa00bd356043078b1d8fdb3e415837, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/4196dc98e4694744be8e34a53b037c1c] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=124.3 K 2024-11-10T15:16:49,382 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:49,382 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/8d3eba0879ea422ca53958c3a520acc4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/edfa00bd356043078b1d8fdb3e415837, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/4196dc98e4694744be8e34a53b037c1c] 2024-11-10T15:16:49,383 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:49,383 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/B is initiating minor compaction (all files) 2024-11-10T15:16:49,383 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/B in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:49,383 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/618e9e669b8945b689fbdd18c48ae688, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e0f12cc24bc845518f2f23b1ca64a623, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/584209bbb9a2470e8211a0461daf0e18] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=35.2 K 2024-11-10T15:16:49,383 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d3eba0879ea422ca53958c3a520acc4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731251808223 2024-11-10T15:16:49,384 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 618e9e669b8945b689fbdd18c48ae688, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731251808240 2024-11-10T15:16:49,384 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting edfa00bd356043078b1d8fdb3e415837, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731251808334 2024-11-10T15:16:49,385 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting e0f12cc24bc845518f2f23b1ca64a623, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731251808334 2024-11-10T15:16:49,385 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4196dc98e4694744be8e34a53b037c1c, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731251808692 2024-11-10T15:16:49,386 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 584209bbb9a2470e8211a0461daf0e18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731251808696 2024-11-10T15:16:49,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:16:49,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:49,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:16:49,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:49,407 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#B#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:49,410 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/dab37d7370974669a06493e1306d2a59 is 50, key is test_row_0/B:col10/1731251808719/Put/seqid=0 2024-11-10T15:16:49,417 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:49,425 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110efc28d04f46d4122a5a3d014ff8cf9c2_7c1cb26053f9cb07691f27c558e31c12 store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:49,435 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110efc28d04f46d4122a5a3d014ff8cf9c2_7c1cb26053f9cb07691f27c558e31c12, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:49,436 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110efc28d04f46d4122a5a3d014ff8cf9c2_7c1cb26053f9cb07691f27c558e31c12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:49,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741948_1124 (size=12104) 2024-11-10T15:16:49,462 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/dab37d7370974669a06493e1306d2a59 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/dab37d7370974669a06493e1306d2a59 2024-11-10T15:16:49,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741949_1125 (size=4469) 2024-11-10T15:16:49,470 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/B of 7c1cb26053f9cb07691f27c558e31c12 into dab37d7370974669a06493e1306d2a59(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:49,470 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:49,471 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/B, priority=13, startTime=1731251809380; duration=0sec 2024-11-10T15:16:49,471 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:49,471 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:B 2024-11-10T15:16:49,471 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T15:16:49,473 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#A#compaction#106 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:49,473 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:49,473 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/C is initiating minor compaction (all files) 2024-11-10T15:16:49,473 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/C in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:49,473 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/e0f2e06a8c594a62a61f0ea8b8503f4f is 175, key is test_row_0/A:col10/1731251808719/Put/seqid=0 2024-11-10T15:16:49,474 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/6b68ae09e5c64517abd50edaeac6a598, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d3ebe27c94674e05a05ba4ae1757b0a2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/23ac4329a03e44b98c48bd8bd4a28ed4] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=35.2 K 2024-11-10T15:16:49,475 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b68ae09e5c64517abd50edaeac6a598, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731251808240 2024-11-10T15:16:49,476 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting d3ebe27c94674e05a05ba4ae1757b0a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731251808334 2024-11-10T15:16:49,481 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 23ac4329a03e44b98c48bd8bd4a28ed4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731251808696 2024-11-10T15:16:49,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T15:16:49,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:49,482 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-10T15:16:49,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251808799, max timestamp=1731251809480Num uniques 859; 2024-11-10T15:16:49,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:16:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:49,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:16:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:16:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:16:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:16:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:16:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:16:49,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741950_1126 (size=31058) 2024-11-10T15:16:49,532 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#C#compaction#107 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:49,534 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/2a7ca3b0ff1a4e2ca16d2128352bd2f6 is 50, key is test_row_0/C:col10/1731251808719/Put/seqid=0 2024-11-10T15:16:49,537 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/e0f2e06a8c594a62a61f0ea8b8503f4f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/e0f2e06a8c594a62a61f0ea8b8503f4f 2024-11-10T15:16:49,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111030222bfd494349cd9e850d7e3bd61e11_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251809480/Put/seqid=0 2024-11-10T15:16:49,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741951_1127 (size=12104) 2024-11-10T15:16:49,560 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/A of 7c1cb26053f9cb07691f27c558e31c12 into e0f2e06a8c594a62a61f0ea8b8503f4f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:49,560 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:49,560 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/A, priority=13, startTime=1731251809379; duration=0sec 2024-11-10T15:16:49,569 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:49,569 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:A 2024-11-10T15:16:49,576 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/2a7ca3b0ff1a4e2ca16d2128352bd2f6 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2a7ca3b0ff1a4e2ca16d2128352bd2f6 2024-11-10T15:16:49,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741952_1128 (size=14594) 2024-11-10T15:16:49,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:49,597 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111030222bfd494349cd9e850d7e3bd61e11_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111030222bfd494349cd9e850d7e3bd61e11_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:49,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251869567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/991bac5da9dc43c684686e3b1c9da19b, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:49,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/991bac5da9dc43c684686e3b1c9da19b is 175, key is test_row_0/A:col10/1731251809480/Put/seqid=0 2024-11-10T15:16:49,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251869590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251869592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251869601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251869601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,609 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/C of 7c1cb26053f9cb07691f27c558e31c12 into 2a7ca3b0ff1a4e2ca16d2128352bd2f6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:49,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,613 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,613 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/C, priority=13, startTime=1731251809396; duration=0sec 2024-11-10T15:16:49,614 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:49,614 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:C 2024-11-10T15:16:49,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741953_1129 (size=39549) 2024-11-10T15:16:49,635 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/991bac5da9dc43c684686e3b1c9da19b 2024-11-10T15:16:49,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/20c4fb74db3e4862a0f2f40f9d8fa23f is 50, key is test_row_0/B:col10/1731251809480/Put/seqid=0 2024-11-10T15:16:49,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741954_1130 (size=12001) 2024-11-10T15:16:49,692 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/20c4fb74db3e4862a0f2f40f9d8fa23f 2024-11-10T15:16:49,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251869714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251869713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/c238a2217c2f423c96fd8ccc1df39c3d is 50, key is test_row_0/C:col10/1731251809480/Put/seqid=0 2024-11-10T15:16:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251869724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251869724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251869724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741955_1131 (size=12001) 2024-11-10T15:16:49,769 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/c238a2217c2f423c96fd8ccc1df39c3d 2024-11-10T15:16:49,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/991bac5da9dc43c684686e3b1c9da19b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/991bac5da9dc43c684686e3b1c9da19b 2024-11-10T15:16:49,787 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/991bac5da9dc43c684686e3b1c9da19b, entries=200, sequenceid=84, filesize=38.6 K 2024-11-10T15:16:49,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/20c4fb74db3e4862a0f2f40f9d8fa23f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/20c4fb74db3e4862a0f2f40f9d8fa23f 2024-11-10T15:16:49,800 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/20c4fb74db3e4862a0f2f40f9d8fa23f, entries=150, sequenceid=84, filesize=11.7 K 2024-11-10T15:16:49,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/c238a2217c2f423c96fd8ccc1df39c3d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c238a2217c2f423c96fd8ccc1df39c3d 2024-11-10T15:16:49,813 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c238a2217c2f423c96fd8ccc1df39c3d, entries=150, sequenceid=84, filesize=11.7 K 2024-11-10T15:16:49,815 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 7c1cb26053f9cb07691f27c558e31c12 in 333ms, sequenceid=84, compaction requested=false 2024-11-10T15:16:49,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:49,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:49,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-10T15:16:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-10T15:16:49,821 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-10T15:16:49,821 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 969 msec 2024-11-10T15:16:49,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 980 msec 2024-11-10T15:16:49,917 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T15:16:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:49,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T15:16:49,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:16:49,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:49,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:16:49,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:49,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:16:49,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:49,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d85c0ef66f9440409dc1c625e1ad5278_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251809933/Put/seqid=0 2024-11-10T15:16:49,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741956_1132 (size=12154) 2024-11-10T15:16:49,964 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:49,970 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d85c0ef66f9440409dc1c625e1ad5278_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d85c0ef66f9440409dc1c625e1ad5278_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:49,972 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/8fe7d16867df45f3b1329eb05fece8cc, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:49,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/8fe7d16867df45f3b1329eb05fece8cc is 175, key is test_row_0/A:col10/1731251809933/Put/seqid=0 2024-11-10T15:16:49,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251869955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251869957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251869970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251869972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251869974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:49,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:49,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T15:16:49,983 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:49,983 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:16:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-10T15:16:49,988 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:16:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T15:16:49,992 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:16:49,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:16:49,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741957_1133 (size=30955) 2024-11-10T15:16:49,996 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/8fe7d16867df45f3b1329eb05fece8cc 2024-11-10T15:16:50,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/00cc213f51664a49ad8d9125721a9da3 is 50, key is test_row_0/B:col10/1731251809933/Put/seqid=0 2024-11-10T15:16:50,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741958_1134 (size=12001) 2024-11-10T15:16:50,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251870081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251870081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251870081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251870092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251870092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T15:16:50,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:50,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251870292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251870292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251870292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:50,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:50,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:50,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:50,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251870302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251870303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T15:16:50,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/00cc213f51664a49ad8d9125721a9da3 2024-11-10T15:16:50,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/e3c9c91e52f44cb1b07c0adc006b9def is 50, key is test_row_0/C:col10/1731251809933/Put/seqid=0 2024-11-10T15:16:50,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:50,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:50,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:50,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:50,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741959_1135 (size=12001) 2024-11-10T15:16:50,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/e3c9c91e52f44cb1b07c0adc006b9def 2024-11-10T15:16:50,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/8fe7d16867df45f3b1329eb05fece8cc as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/8fe7d16867df45f3b1329eb05fece8cc 2024-11-10T15:16:50,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/8fe7d16867df45f3b1329eb05fece8cc, entries=150, sequenceid=104, filesize=30.2 K 2024-11-10T15:16:50,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/00cc213f51664a49ad8d9125721a9da3 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/00cc213f51664a49ad8d9125721a9da3 2024-11-10T15:16:50,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/00cc213f51664a49ad8d9125721a9da3, entries=150, sequenceid=104, filesize=11.7 K 2024-11-10T15:16:50,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/e3c9c91e52f44cb1b07c0adc006b9def as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/e3c9c91e52f44cb1b07c0adc006b9def 2024-11-10T15:16:50,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/e3c9c91e52f44cb1b07c0adc006b9def, entries=150, sequenceid=104, filesize=11.7 K 2024-11-10T15:16:50,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 7c1cb26053f9cb07691f27c558e31c12 in 591ms, sequenceid=104, compaction requested=true 2024-11-10T15:16:50,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:50,527 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:50,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:16:50,532 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:50,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:50,532 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/A is initiating minor compaction (all files) 2024-11-10T15:16:50,532 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:50,532 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/A in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:50,532 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/e0f2e06a8c594a62a61f0ea8b8503f4f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/991bac5da9dc43c684686e3b1c9da19b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/8fe7d16867df45f3b1329eb05fece8cc] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=99.2 K 2024-11-10T15:16:50,532 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:50,532 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/e0f2e06a8c594a62a61f0ea8b8503f4f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/991bac5da9dc43c684686e3b1c9da19b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/8fe7d16867df45f3b1329eb05fece8cc] 2024-11-10T15:16:50,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:16:50,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:50,533 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0f2e06a8c594a62a61f0ea8b8503f4f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731251808696 2024-11-10T15:16:50,533 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 991bac5da9dc43c684686e3b1c9da19b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731251808799 2024-11-10T15:16:50,533 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:50,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:16:50,533 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/B is initiating minor compaction (all files) 2024-11-10T15:16:50,533 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/B in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:50,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:50,534 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/dab37d7370974669a06493e1306d2a59, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/20c4fb74db3e4862a0f2f40f9d8fa23f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/00cc213f51664a49ad8d9125721a9da3] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=35.3 K 2024-11-10T15:16:50,534 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fe7d16867df45f3b1329eb05fece8cc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731251809592 2024-11-10T15:16:50,534 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting dab37d7370974669a06493e1306d2a59, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731251808696 2024-11-10T15:16:50,535 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 20c4fb74db3e4862a0f2f40f9d8fa23f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731251808825 2024-11-10T15:16:50,537 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 00cc213f51664a49ad8d9125721a9da3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731251809592 2024-11-10T15:16:50,565 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:50,572 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110e7e7e98540a34d00b33fb66bd4b3d579_7c1cb26053f9cb07691f27c558e31c12 store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:50,573 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#B#compaction#115 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:50,574 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/f87f196a65bc408f88b71c1ae2088688 is 50, key is test_row_0/B:col10/1731251809933/Put/seqid=0 2024-11-10T15:16:50,579 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110e7e7e98540a34d00b33fb66bd4b3d579_7c1cb26053f9cb07691f27c558e31c12, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:50,580 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e7e7e98540a34d00b33fb66bd4b3d579_7c1cb26053f9cb07691f27c558e31c12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:50,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T15:16:50,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:16:50,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:50,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:16:50,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:50,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:16:50,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:50,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:50,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:50,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:50,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:50,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T15:16:50,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741960_1136 (size=12207) 2024-11-10T15:16:50,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251870648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251870650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251870650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251870655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251870655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741961_1137 (size=4469) 2024-11-10T15:16:50,677 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#A#compaction#114 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:50,678 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/a6eb6565ee0840cda75f7ed4e8f035a0 is 175, key is test_row_0/A:col10/1731251809933/Put/seqid=0 2024-11-10T15:16:50,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f29136ae47ba4e11bf66c71c332ec592_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251809967/Put/seqid=0 2024-11-10T15:16:50,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741962_1138 (size=31161) 2024-11-10T15:16:50,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741963_1139 (size=14594) 2024-11-10T15:16:50,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:50,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:50,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:50,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:50,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251870771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251870771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251870772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251870771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251870773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:50,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:50,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:50,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:50,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:50,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251870991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251870992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251870992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251870992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251870992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:50,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:50,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:50,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:51,042 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/f87f196a65bc408f88b71c1ae2088688 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/f87f196a65bc408f88b71c1ae2088688 2024-11-10T15:16:51,050 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/B of 7c1cb26053f9cb07691f27c558e31c12 into f87f196a65bc408f88b71c1ae2088688(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:51,051 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:51,051 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/B, priority=13, startTime=1731251810532; duration=0sec 2024-11-10T15:16:51,051 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:51,051 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:B 2024-11-10T15:16:51,051 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:51,053 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:51,053 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/C is initiating minor compaction (all files) 2024-11-10T15:16:51,053 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/C in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:51,053 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2a7ca3b0ff1a4e2ca16d2128352bd2f6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c238a2217c2f423c96fd8ccc1df39c3d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/e3c9c91e52f44cb1b07c0adc006b9def] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=35.3 K 2024-11-10T15:16:51,054 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a7ca3b0ff1a4e2ca16d2128352bd2f6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731251808696 2024-11-10T15:16:51,057 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting c238a2217c2f423c96fd8ccc1df39c3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731251808825 2024-11-10T15:16:51,059 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting e3c9c91e52f44cb1b07c0adc006b9def, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731251809592 2024-11-10T15:16:51,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:51,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:51,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:51,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:51,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,091 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#C#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:51,091 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/9f467555ed2a4c6fb5898059a457082f is 50, key is test_row_0/C:col10/1731251809933/Put/seqid=0 2024-11-10T15:16:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T15:16:51,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741964_1140 (size=12207) 2024-11-10T15:16:51,159 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/a6eb6565ee0840cda75f7ed4e8f035a0 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a6eb6565ee0840cda75f7ed4e8f035a0 2024-11-10T15:16:51,160 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:51,167 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f29136ae47ba4e11bf66c71c332ec592_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f29136ae47ba4e11bf66c71c332ec592_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:51,168 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/A of 7c1cb26053f9cb07691f27c558e31c12 into a6eb6565ee0840cda75f7ed4e8f035a0(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:51,169 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:51,169 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/A, priority=13, startTime=1731251810527; duration=0sec 2024-11-10T15:16:51,169 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/a8d0de89a7704165a203ca8bda4b0250, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:51,169 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:51,169 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:A 2024-11-10T15:16:51,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/a8d0de89a7704165a203ca8bda4b0250 is 175, key is test_row_0/A:col10/1731251809967/Put/seqid=0 2024-11-10T15:16:51,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741965_1141 (size=39549) 2024-11-10T15:16:51,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:51,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:51,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:51,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:51,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251871301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:51,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:51,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:51,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251871301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251871301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:51,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251871312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:51,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251871312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:51,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:51,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:51,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:51,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:51,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:51,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:51,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:51,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:51,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,548 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/9f467555ed2a4c6fb5898059a457082f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/9f467555ed2a4c6fb5898059a457082f 2024-11-10T15:16:51,572 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/C of 7c1cb26053f9cb07691f27c558e31c12 into 9f467555ed2a4c6fb5898059a457082f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:51,572 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:51,572 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/C, priority=13, startTime=1731251810533; duration=0sec 2024-11-10T15:16:51,572 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:51,572 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:C 2024-11-10T15:16:51,624 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/a8d0de89a7704165a203ca8bda4b0250 2024-11-10T15:16:51,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/e4bb914ea7ae49cf87a231eab9fbd2fe is 50, key is test_row_0/B:col10/1731251809967/Put/seqid=0 2024-11-10T15:16:51,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741966_1142 (size=12001) 2024-11-10T15:16:51,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:51,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:51,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:51,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:51,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251871812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:51,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:51,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251871812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251871821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:51,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251871822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:51,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251871823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:51,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:51,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:51,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:51,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:51,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:51,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:51,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:52,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:52,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:52,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:52,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/e4bb914ea7ae49cf87a231eab9fbd2fe 2024-11-10T15:16:52,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/b4ceffe8a006492e8c4b3d2c307f7e8e is 50, key is test_row_0/C:col10/1731251809967/Put/seqid=0 2024-11-10T15:16:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T15:16:52,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741967_1143 (size=12001) 2024-11-10T15:16:52,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:52,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:52,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:52,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:52,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:52,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:52,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:52,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:52,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:52,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:52,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:52,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:52,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:52,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:52,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:52,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:52,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:52,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:52,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/b4ceffe8a006492e8c4b3d2c307f7e8e 2024-11-10T15:16:52,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/a8d0de89a7704165a203ca8bda4b0250 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a8d0de89a7704165a203ca8bda4b0250 2024-11-10T15:16:52,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a8d0de89a7704165a203ca8bda4b0250, entries=200, sequenceid=119, filesize=38.6 K 2024-11-10T15:16:52,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/e4bb914ea7ae49cf87a231eab9fbd2fe as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e4bb914ea7ae49cf87a231eab9fbd2fe 2024-11-10T15:16:52,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e4bb914ea7ae49cf87a231eab9fbd2fe, entries=150, sequenceid=119, filesize=11.7 K 2024-11-10T15:16:52,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/b4ceffe8a006492e8c4b3d2c307f7e8e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/b4ceffe8a006492e8c4b3d2c307f7e8e 2024-11-10T15:16:52,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/b4ceffe8a006492e8c4b3d2c307f7e8e, entries=150, sequenceid=119, filesize=11.7 K 2024-11-10T15:16:52,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 7c1cb26053f9cb07691f27c558e31c12 in 2008ms, sequenceid=119, compaction requested=false 2024-11-10T15:16:52,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:52,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T15:16:52,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:52,647 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T15:16:52,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:16:52,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:52,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:16:52,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:52,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:16:52,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:52,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110781f3364a8f248c397337534763531c3_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251810624/Put/seqid=0 2024-11-10T15:16:52,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741968_1144 (size=12304) 2024-11-10T15:16:52,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:52,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251872852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:52,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:52,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251872852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:52,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251872864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251872868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:52,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251872868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:52,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:52,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251872971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:52,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:52,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251872982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:52,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251872982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251872982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251872982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:53,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:53,164 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110781f3364a8f248c397337534763531c3_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110781f3364a8f248c397337534763531c3_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:53,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/3fc20983e6d7429ba632c50955a98dca, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:53,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/3fc20983e6d7429ba632c50955a98dca is 175, key is test_row_0/A:col10/1731251810624/Put/seqid=0 2024-11-10T15:16:53,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:53,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251873182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:53,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:53,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251873192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:53,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251873192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:53,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:53,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:53,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251873192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:53,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:53,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251873196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:53,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:53,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741969_1145 (size=31105) 2024-11-10T15:16:53,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251873505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:53,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251873505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:53,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251873505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:53,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251873505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:53,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251873512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:53,610 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/3fc20983e6d7429ba632c50955a98dca 2024-11-10T15:16:53,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/e053615d3c104d998bfeec9b145d9c86 is 50, key is test_row_0/B:col10/1731251810624/Put/seqid=0 2024-11-10T15:16:53,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741970_1146 (size=12151) 2024-11-10T15:16:53,693 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/e053615d3c104d998bfeec9b145d9c86 2024-11-10T15:16:53,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/344f26407b534b309ffa9153713ff408 is 50, key is test_row_0/C:col10/1731251810624/Put/seqid=0 2024-11-10T15:16:53,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741971_1147 (size=12151) 2024-11-10T15:16:53,744 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/344f26407b534b309ffa9153713ff408 2024-11-10T15:16:53,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/3fc20983e6d7429ba632c50955a98dca as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/3fc20983e6d7429ba632c50955a98dca 2024-11-10T15:16:53,776 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/3fc20983e6d7429ba632c50955a98dca, entries=150, sequenceid=140, filesize=30.4 K 2024-11-10T15:16:53,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/e053615d3c104d998bfeec9b145d9c86 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e053615d3c104d998bfeec9b145d9c86 2024-11-10T15:16:53,796 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e053615d3c104d998bfeec9b145d9c86, entries=150, sequenceid=140, filesize=11.9 K 2024-11-10T15:16:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/344f26407b534b309ffa9153713ff408 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/344f26407b534b309ffa9153713ff408 2024-11-10T15:16:53,825 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/344f26407b534b309ffa9153713ff408, entries=150, sequenceid=140, filesize=11.9 K 2024-11-10T15:16:53,826 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 7c1cb26053f9cb07691f27c558e31c12 in 1180ms, sequenceid=140, compaction requested=true 2024-11-10T15:16:53,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:53,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:53,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-10T15:16:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-10T15:16:53,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-10T15:16:53,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8380 sec 2024-11-10T15:16:53,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 3.8530 sec 2024-11-10T15:16:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:54,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T15:16:54,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:16:54,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:54,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:16:54,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:54,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:16:54,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:54,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102b8f993073b84e50ab71062646f6c108_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251814026/Put/seqid=0 2024-11-10T15:16:54,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251874065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251874070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251874080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251874083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251874086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741972_1148 (size=17284) 2024-11-10T15:16:54,095 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:54,102 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102b8f993073b84e50ab71062646f6c108_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102b8f993073b84e50ab71062646f6c108_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:54,103 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/628e995c68f34be5a8368170a121c88f, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:54,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/628e995c68f34be5a8368170a121c88f is 175, key is test_row_0/A:col10/1731251814026/Put/seqid=0 2024-11-10T15:16:54,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741973_1149 (size=48389) 2024-11-10T15:16:54,152 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/628e995c68f34be5a8368170a121c88f 2024-11-10T15:16:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T15:16:54,163 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:54,166 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:16:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-10T15:16:54,169 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:16:54,170 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:16:54,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:16:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T15:16:54,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/2d4a62edb8cb4aa3a82d5f8e6995e619 is 50, key is test_row_0/B:col10/1731251814026/Put/seqid=0 2024-11-10T15:16:54,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251874191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741974_1150 (size=12151) 2024-11-10T15:16:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251874191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251874194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251874194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/2d4a62edb8cb4aa3a82d5f8e6995e619 2024-11-10T15:16:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251874202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d4a2ad61ccaf4e9a846bb2b4bfcdb848 is 50, key is test_row_0/C:col10/1731251814026/Put/seqid=0 2024-11-10T15:16:54,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741975_1151 (size=12151) 2024-11-10T15:16:54,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d4a2ad61ccaf4e9a846bb2b4bfcdb848 2024-11-10T15:16:54,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/628e995c68f34be5a8368170a121c88f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/628e995c68f34be5a8368170a121c88f 2024-11-10T15:16:54,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/628e995c68f34be5a8368170a121c88f, entries=250, sequenceid=156, filesize=47.3 K 2024-11-10T15:16:54,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/2d4a62edb8cb4aa3a82d5f8e6995e619 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/2d4a62edb8cb4aa3a82d5f8e6995e619 2024-11-10T15:16:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T15:16:54,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/2d4a62edb8cb4aa3a82d5f8e6995e619, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T15:16:54,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d4a2ad61ccaf4e9a846bb2b4bfcdb848 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d4a2ad61ccaf4e9a846bb2b4bfcdb848 2024-11-10T15:16:54,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d4a2ad61ccaf4e9a846bb2b4bfcdb848, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T15:16:54,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 7c1cb26053f9cb07691f27c558e31c12 in 269ms, sequenceid=156, compaction requested=true 2024-11-10T15:16:54,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:54,296 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:16:54,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:16:54,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:54,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:16:54,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:54,296 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:16:54,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:16:54,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:54,298 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150204 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:16:54,298 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/A is initiating minor compaction (all files) 2024-11-10T15:16:54,298 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/A in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:54,298 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a6eb6565ee0840cda75f7ed4e8f035a0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a8d0de89a7704165a203ca8bda4b0250, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/3fc20983e6d7429ba632c50955a98dca, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/628e995c68f34be5a8368170a121c88f] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=146.7 K 2024-11-10T15:16:54,298 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:54,298 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a6eb6565ee0840cda75f7ed4e8f035a0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a8d0de89a7704165a203ca8bda4b0250, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/3fc20983e6d7429ba632c50955a98dca, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/628e995c68f34be5a8368170a121c88f] 2024-11-10T15:16:54,298 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:16:54,299 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/B is initiating minor compaction (all files) 2024-11-10T15:16:54,299 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/B in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:54,299 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/f87f196a65bc408f88b71c1ae2088688, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e4bb914ea7ae49cf87a231eab9fbd2fe, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e053615d3c104d998bfeec9b145d9c86, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/2d4a62edb8cb4aa3a82d5f8e6995e619] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=47.4 K 2024-11-10T15:16:54,301 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting f87f196a65bc408f88b71c1ae2088688, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731251809592 2024-11-10T15:16:54,301 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6eb6565ee0840cda75f7ed4e8f035a0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731251809592 2024-11-10T15:16:54,307 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8d0de89a7704165a203ca8bda4b0250, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731251809967 2024-11-10T15:16:54,307 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting e4bb914ea7ae49cf87a231eab9fbd2fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731251809967 2024-11-10T15:16:54,309 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting e053615d3c104d998bfeec9b145d9c86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731251810624 2024-11-10T15:16:54,309 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fc20983e6d7429ba632c50955a98dca, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731251810624 2024-11-10T15:16:54,310 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 628e995c68f34be5a8368170a121c88f, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731251812846 2024-11-10T15:16:54,311 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d4a62edb8cb4aa3a82d5f8e6995e619, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731251812846 2024-11-10T15:16:54,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T15:16:54,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:54,325 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T15:16:54,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:16:54,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:54,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:16:54,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:16:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:54,331 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:54,333 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#B#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:54,334 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/cee73babd8fb4d6190eb1d9e07ac87c0 is 50, key is test_row_0/B:col10/1731251814026/Put/seqid=0 2024-11-10T15:16:54,371 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110fd2c5dde3c7e4a9c9b3d5a7774ccd704_7c1cb26053f9cb07691f27c558e31c12 store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:54,375 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110fd2c5dde3c7e4a9c9b3d5a7774ccd704_7c1cb26053f9cb07691f27c558e31c12, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:54,375 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fd2c5dde3c7e4a9c9b3d5a7774ccd704_7c1cb26053f9cb07691f27c558e31c12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:54,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741976_1152 (size=12493) 2024-11-10T15:16:54,401 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/cee73babd8fb4d6190eb1d9e07ac87c0 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cee73babd8fb4d6190eb1d9e07ac87c0 2024-11-10T15:16:54,408 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/B of 7c1cb26053f9cb07691f27c558e31c12 into cee73babd8fb4d6190eb1d9e07ac87c0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:54,409 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:54,409 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/B, priority=12, startTime=1731251814296; duration=0sec 2024-11-10T15:16:54,409 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:54,409 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:B 2024-11-10T15:16:54,409 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:16:54,411 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:16:54,411 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/C is initiating minor compaction (all files) 2024-11-10T15:16:54,411 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/C in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:54,411 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/9f467555ed2a4c6fb5898059a457082f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/b4ceffe8a006492e8c4b3d2c307f7e8e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/344f26407b534b309ffa9153713ff408, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d4a2ad61ccaf4e9a846bb2b4bfcdb848] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=47.4 K 2024-11-10T15:16:54,412 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f467555ed2a4c6fb5898059a457082f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731251809592 2024-11-10T15:16:54,413 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting b4ceffe8a006492e8c4b3d2c307f7e8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731251809967 2024-11-10T15:16:54,414 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 344f26407b534b309ffa9153713ff408, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731251810624 2024-11-10T15:16:54,414 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting d4a2ad61ccaf4e9a846bb2b4bfcdb848, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731251812846 2024-11-10T15:16:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:54,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:54,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e945b0e740cf42299d52dd21738d3b96_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251814060/Put/seqid=0 2024-11-10T15:16:54,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741977_1153 (size=4469) 2024-11-10T15:16:54,470 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#A#compaction#127 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:54,471 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/0a5100b275464fd5a3be8e7bae39e4f1 is 175, key is test_row_0/A:col10/1731251814026/Put/seqid=0 2024-11-10T15:16:54,474 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#C#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:54,475 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/1b792d90406e46a49cfc24b964493b2f is 50, key is test_row_0/C:col10/1731251814026/Put/seqid=0 2024-11-10T15:16:54,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251874462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251874463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251874469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251874470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251874475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T15:16:54,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741978_1154 (size=12304) 2024-11-10T15:16:54,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:54,534 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e945b0e740cf42299d52dd21738d3b96_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e945b0e740cf42299d52dd21738d3b96_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:54,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/4c251a52d2e24dcea7055c54c9bfbd78, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:54,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/4c251a52d2e24dcea7055c54c9bfbd78 is 175, key is test_row_0/A:col10/1731251814060/Put/seqid=0 2024-11-10T15:16:54,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741980_1156 (size=12493) 2024-11-10T15:16:54,563 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/1b792d90406e46a49cfc24b964493b2f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/1b792d90406e46a49cfc24b964493b2f 2024-11-10T15:16:54,583 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/C of 7c1cb26053f9cb07691f27c558e31c12 into 1b792d90406e46a49cfc24b964493b2f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:54,583 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:54,583 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/C, priority=12, startTime=1731251814296; duration=0sec 2024-11-10T15:16:54,583 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:54,584 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:C 2024-11-10T15:16:54,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251874583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251874583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251874583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251874582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741979_1155 (size=31447) 2024-11-10T15:16:54,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251874591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,604 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/0a5100b275464fd5a3be8e7bae39e4f1 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/0a5100b275464fd5a3be8e7bae39e4f1 2024-11-10T15:16:54,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,618 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/A of 7c1cb26053f9cb07691f27c558e31c12 into 0a5100b275464fd5a3be8e7bae39e4f1(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:54,618 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:54,618 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/A, priority=12, startTime=1731251814296; duration=0sec 2024-11-10T15:16:54,618 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:54,618 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:A 2024-11-10T15:16:54,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741981_1157 (size=31105) 2024-11-10T15:16:54,642 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/4c251a52d2e24dcea7055c54c9bfbd78 2024-11-10T15:16:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/cdfc6e5c6f2e49eeb92460a9b76d0405 is 50, key is test_row_0/B:col10/1731251814060/Put/seqid=0 2024-11-10T15:16:54,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741982_1158 (size=12151) 2024-11-10T15:16:54,709 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/cdfc6e5c6f2e49eeb92460a9b76d0405 2024-11-10T15:16:54,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/2ed27ce5852343e1ad9fe81c3b4ffdb9 is 50, key is test_row_0/C:col10/1731251814060/Put/seqid=0 2024-11-10T15:16:54,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741983_1159 (size=12151) 2024-11-10T15:16:54,779 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/2ed27ce5852343e1ad9fe81c3b4ffdb9 2024-11-10T15:16:54,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/4c251a52d2e24dcea7055c54c9bfbd78 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/4c251a52d2e24dcea7055c54c9bfbd78 2024-11-10T15:16:54,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251874791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251874792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T15:16:54,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251874803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,806 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/4c251a52d2e24dcea7055c54c9bfbd78, entries=150, sequenceid=173, filesize=30.4 K 2024-11-10T15:16:54,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/cdfc6e5c6f2e49eeb92460a9b76d0405 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cdfc6e5c6f2e49eeb92460a9b76d0405 2024-11-10T15:16:54,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251874804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,820 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cdfc6e5c6f2e49eeb92460a9b76d0405, entries=150, sequenceid=173, filesize=11.9 K 2024-11-10T15:16:54,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/2ed27ce5852343e1ad9fe81c3b4ffdb9 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2ed27ce5852343e1ad9fe81c3b4ffdb9 2024-11-10T15:16:54,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251874825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:54,857 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2ed27ce5852343e1ad9fe81c3b4ffdb9, entries=150, sequenceid=173, filesize=11.9 K 2024-11-10T15:16:54,858 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 7c1cb26053f9cb07691f27c558e31c12 in 533ms, sequenceid=173, compaction requested=false 2024-11-10T15:16:54,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:54,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:54,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-10T15:16:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-10T15:16:54,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-10T15:16:54,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 696 msec 2024-11-10T15:16:54,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 704 msec 2024-11-10T15:16:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:55,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T15:16:55,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:16:55,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:55,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:16:55,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:55,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:16:55,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:55,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b5c1cd1e1e3c452c879334710d5214fb_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251814455/Put/seqid=0 2024-11-10T15:16:55,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251875150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251875150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251875151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251875152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251875159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741984_1160 (size=14794) 2024-11-10T15:16:55,190 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:55,203 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b5c1cd1e1e3c452c879334710d5214fb_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b5c1cd1e1e3c452c879334710d5214fb_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:55,205 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/61e58ae9f9884835878646b727572ff8, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:55,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/61e58ae9f9884835878646b727572ff8 is 175, key is test_row_0/A:col10/1731251814455/Put/seqid=0 2024-11-10T15:16:55,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741985_1161 (size=39749) 2024-11-10T15:16:55,234 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/61e58ae9f9884835878646b727572ff8 2024-11-10T15:16:55,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/270b235ba4964176a2dc3088485d5faf is 50, key is test_row_0/B:col10/1731251814455/Put/seqid=0 2024-11-10T15:16:55,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741986_1162 (size=12151) 2024-11-10T15:16:55,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/270b235ba4964176a2dc3088485d5faf 2024-11-10T15:16:55,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251875271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251875271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251875271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251875272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/ee29dcb294c1417daefe133e1132a8ba is 50, key is test_row_0/C:col10/1731251814455/Put/seqid=0 2024-11-10T15:16:55,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741987_1163 (size=12151) 2024-11-10T15:16:55,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/ee29dcb294c1417daefe133e1132a8ba 2024-11-10T15:16:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T15:16:55,312 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:55,314 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:16:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-10T15:16:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T15:16:55,316 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:16:55,317 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:16:55,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:16:55,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/61e58ae9f9884835878646b727572ff8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/61e58ae9f9884835878646b727572ff8 2024-11-10T15:16:55,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/61e58ae9f9884835878646b727572ff8, entries=200, sequenceid=192, filesize=38.8 K 2024-11-10T15:16:55,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/270b235ba4964176a2dc3088485d5faf as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/270b235ba4964176a2dc3088485d5faf 2024-11-10T15:16:55,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/270b235ba4964176a2dc3088485d5faf, entries=150, sequenceid=192, filesize=11.9 K 2024-11-10T15:16:55,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/ee29dcb294c1417daefe133e1132a8ba as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/ee29dcb294c1417daefe133e1132a8ba 2024-11-10T15:16:55,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/ee29dcb294c1417daefe133e1132a8ba, entries=150, sequenceid=192, filesize=11.9 K 2024-11-10T15:16:55,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7c1cb26053f9cb07691f27c558e31c12 in 292ms, sequenceid=192, compaction requested=true 2024-11-10T15:16:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:16:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:55,398 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:55,399 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:16:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:16:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:55,400 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:55,400 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/B is initiating minor compaction (all files) 2024-11-10T15:16:55,400 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/B in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:55,400 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:55,400 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cee73babd8fb4d6190eb1d9e07ac87c0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cdfc6e5c6f2e49eeb92460a9b76d0405, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/270b235ba4964176a2dc3088485d5faf] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=35.9 K 2024-11-10T15:16:55,400 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/A is initiating minor compaction (all files) 2024-11-10T15:16:55,400 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/A in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:55,400 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/0a5100b275464fd5a3be8e7bae39e4f1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/4c251a52d2e24dcea7055c54c9bfbd78, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/61e58ae9f9884835878646b727572ff8] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=99.9 K 2024-11-10T15:16:55,400 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:55,400 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/0a5100b275464fd5a3be8e7bae39e4f1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/4c251a52d2e24dcea7055c54c9bfbd78, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/61e58ae9f9884835878646b727572ff8] 2024-11-10T15:16:55,401 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting cee73babd8fb4d6190eb1d9e07ac87c0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731251812846 2024-11-10T15:16:55,401 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a5100b275464fd5a3be8e7bae39e4f1, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731251812846 2024-11-10T15:16:55,402 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting cdfc6e5c6f2e49eeb92460a9b76d0405, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731251814060 2024-11-10T15:16:55,402 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c251a52d2e24dcea7055c54c9bfbd78, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731251814060 2024-11-10T15:16:55,402 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61e58ae9f9884835878646b727572ff8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731251814455 2024-11-10T15:16:55,402 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 270b235ba4964176a2dc3088485d5faf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731251814455 2024-11-10T15:16:55,417 INFO [master/c0771061be61:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T15:16:55,417 INFO [master/c0771061be61:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T15:16:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T15:16:55,438 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:55,446 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#B#compaction#136 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:55,447 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/efa7ebb54d674fdaa739caba0ced0d7a is 50, key is test_row_0/B:col10/1731251814455/Put/seqid=0 2024-11-10T15:16:55,457 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411107b26a36c526d436da16714ec62ded7af_7c1cb26053f9cb07691f27c558e31c12 store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:55,459 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411107b26a36c526d436da16714ec62ded7af_7c1cb26053f9cb07691f27c558e31c12, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:55,459 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107b26a36c526d436da16714ec62ded7af_7c1cb26053f9cb07691f27c558e31c12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:55,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T15:16:55,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:55,479 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T15:16:55,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:16:55,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:55,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:16:55,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:55,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:16:55,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:55,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:55,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741988_1164 (size=12595) 2024-11-10T15:16:55,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741989_1165 (size=4469) 2024-11-10T15:16:55,520 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#A#compaction#135 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:55,520 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/70353ecee1c347119090ca62180f8152 is 175, key is test_row_0/A:col10/1731251814455/Put/seqid=0 2024-11-10T15:16:55,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110512d4f445d5b4bdeb4c88d27c7f0f4d9_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251815150/Put/seqid=0 2024-11-10T15:16:55,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741990_1166 (size=31549) 2024-11-10T15:16:55,541 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/70353ecee1c347119090ca62180f8152 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/70353ecee1c347119090ca62180f8152 2024-11-10T15:16:55,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741991_1167 (size=12304) 2024-11-10T15:16:55,560 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/A of 7c1cb26053f9cb07691f27c558e31c12 into 70353ecee1c347119090ca62180f8152(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:55,560 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:55,560 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/A, priority=13, startTime=1731251815398; duration=0sec 2024-11-10T15:16:55,560 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:55,560 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:A 2024-11-10T15:16:55,560 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:16:55,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:55,563 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:16:55,563 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/C is initiating minor compaction (all files) 2024-11-10T15:16:55,563 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/C in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:55,563 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/1b792d90406e46a49cfc24b964493b2f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2ed27ce5852343e1ad9fe81c3b4ffdb9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/ee29dcb294c1417daefe133e1132a8ba] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=35.9 K 2024-11-10T15:16:55,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251875537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251875537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,566 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b792d90406e46a49cfc24b964493b2f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731251812846 2024-11-10T15:16:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251875542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,572 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ed27ce5852343e1ad9fe81c3b4ffdb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731251814060 2024-11-10T15:16:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,572 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee29dcb294c1417daefe133e1132a8ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731251814455 2024-11-10T15:16:55,577 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110512d4f445d5b4bdeb4c88d27c7f0f4d9_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110512d4f445d5b4bdeb4c88d27c7f0f4d9_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:55,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/508491b4d04e4705a3e2e9cfcac12460, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:55,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/508491b4d04e4705a3e2e9cfcac12460 is 175, key is test_row_0/A:col10/1731251815150/Put/seqid=0 2024-11-10T15:16:55,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251875566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,613 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:55,613 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/c0fa1d359e3749cf9a2b61e25a9ddc42 is 50, key is test_row_0/C:col10/1731251814455/Put/seqid=0 2024-11-10T15:16:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T15:16:55,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741992_1168 (size=31105) 2024-11-10T15:16:55,657 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/508491b4d04e4705a3e2e9cfcac12460 2024-11-10T15:16:55,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251875671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251875672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251875672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741993_1169 (size=12595) 2024-11-10T15:16:55,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251875681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,689 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/c0fa1d359e3749cf9a2b61e25a9ddc42 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c0fa1d359e3749cf9a2b61e25a9ddc42 2024-11-10T15:16:55,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/cc0761912f35488387cd878492fb785e is 50, key is test_row_0/B:col10/1731251815150/Put/seqid=0 2024-11-10T15:16:55,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251875703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,726 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/C of 7c1cb26053f9cb07691f27c558e31c12 into c0fa1d359e3749cf9a2b61e25a9ddc42(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:55,726 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:55,726 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/C, priority=13, startTime=1731251815399; duration=0sec 2024-11-10T15:16:55,726 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:55,726 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:C 2024-11-10T15:16:55,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741994_1170 (size=12151) 2024-11-10T15:16:55,759 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/cc0761912f35488387cd878492fb785e 2024-11-10T15:16:55,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/36f901fde6e846e3af65fec7b1fdf02a is 50, key is test_row_0/C:col10/1731251815150/Put/seqid=0 2024-11-10T15:16:55,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741995_1171 (size=12151) 2024-11-10T15:16:55,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251875881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251875889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251875892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,914 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/efa7ebb54d674fdaa739caba0ced0d7a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/efa7ebb54d674fdaa739caba0ced0d7a 2024-11-10T15:16:55,929 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/B of 7c1cb26053f9cb07691f27c558e31c12 into efa7ebb54d674fdaa739caba0ced0d7a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:55,929 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:55,929 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/B, priority=13, startTime=1731251815398; duration=0sec 2024-11-10T15:16:55,930 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:55,930 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:B 2024-11-10T15:16:55,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251875912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T15:16:56,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251876204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:56,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251876208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:56,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251876211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:56,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:56,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251876242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:56,253 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/36f901fde6e846e3af65fec7b1fdf02a 2024-11-10T15:16:56,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/508491b4d04e4705a3e2e9cfcac12460 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/508491b4d04e4705a3e2e9cfcac12460 2024-11-10T15:16:56,269 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/508491b4d04e4705a3e2e9cfcac12460, entries=150, sequenceid=208, filesize=30.4 K 2024-11-10T15:16:56,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/cc0761912f35488387cd878492fb785e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cc0761912f35488387cd878492fb785e 2024-11-10T15:16:56,279 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cc0761912f35488387cd878492fb785e, entries=150, sequenceid=208, filesize=11.9 K 2024-11-10T15:16:56,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/36f901fde6e846e3af65fec7b1fdf02a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/36f901fde6e846e3af65fec7b1fdf02a 2024-11-10T15:16:56,291 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/36f901fde6e846e3af65fec7b1fdf02a, entries=150, sequenceid=208, filesize=11.9 K 2024-11-10T15:16:56,293 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7c1cb26053f9cb07691f27c558e31c12 in 814ms, sequenceid=208, compaction requested=false 2024-11-10T15:16:56,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:56,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:56,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-10T15:16:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-10T15:16:56,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-10T15:16:56,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 978 msec 2024-11-10T15:16:56,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 984 msec 2024-11-10T15:16:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T15:16:56,452 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:56,452 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:16:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-10T15:16:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T15:16:56,457 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:16:56,458 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:16:56,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:16:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T15:16:56,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T15:16:56,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:56,612 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T15:16:56,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:16:56,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:56,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:16:56,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:56,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:16:56,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:56,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111043e2c3997da2494eaeaba3a5fefddca6_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251815537/Put/seqid=0 2024-11-10T15:16:56,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741996_1172 (size=12304) 2024-11-10T15:16:56,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:56,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:56,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251876754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251876748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T15:16:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:56,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251876755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251876761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251876761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:56,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251876882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:56,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251876882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:56,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251876882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:56,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251876882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T15:16:57,085 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111043e2c3997da2494eaeaba3a5fefddca6_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111043e2c3997da2494eaeaba3a5fefddca6_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:57,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/40c2207c398b46ceb1face1fb047ddb9, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:57,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/40c2207c398b46ceb1face1fb047ddb9 is 175, key is test_row_0/A:col10/1731251815537/Put/seqid=0 2024-11-10T15:16:57,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251877092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251877093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251877093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251877093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741997_1173 (size=31105) 2024-11-10T15:16:57,135 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/40c2207c398b46ceb1face1fb047ddb9 2024-11-10T15:16:57,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/dfa5af95010b421d899b3749770a9a47 is 50, key is test_row_0/B:col10/1731251815537/Put/seqid=0 2024-11-10T15:16:57,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741998_1174 (size=12151) 2024-11-10T15:16:57,177 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/dfa5af95010b421d899b3749770a9a47 2024-11-10T15:16:57,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/1ca4b9559e6548e5bc3fc6166657b21b is 50, key is test_row_0/C:col10/1731251815537/Put/seqid=0 2024-11-10T15:16:57,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741999_1175 (size=12151) 2024-11-10T15:16:57,241 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/1ca4b9559e6548e5bc3fc6166657b21b 2024-11-10T15:16:57,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/40c2207c398b46ceb1face1fb047ddb9 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/40c2207c398b46ceb1face1fb047ddb9 2024-11-10T15:16:57,257 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/40c2207c398b46ceb1face1fb047ddb9, entries=150, sequenceid=227, filesize=30.4 K 2024-11-10T15:16:57,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/dfa5af95010b421d899b3749770a9a47 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/dfa5af95010b421d899b3749770a9a47 2024-11-10T15:16:57,277 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/dfa5af95010b421d899b3749770a9a47, entries=150, sequenceid=227, filesize=11.9 K 2024-11-10T15:16:57,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/1ca4b9559e6548e5bc3fc6166657b21b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/1ca4b9559e6548e5bc3fc6166657b21b 2024-11-10T15:16:57,294 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/1ca4b9559e6548e5bc3fc6166657b21b, entries=150, sequenceid=227, filesize=11.9 K 2024-11-10T15:16:57,296 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7c1cb26053f9cb07691f27c558e31c12 in 684ms, sequenceid=227, compaction requested=true 2024-11-10T15:16:57,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:57,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:57,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-10T15:16:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-10T15:16:57,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-10T15:16:57,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 846 msec 2024-11-10T15:16:57,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 869 msec 2024-11-10T15:16:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:57,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T15:16:57,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:16:57,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:57,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:16:57,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:57,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:16:57,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:57,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110aa68cf23dc434794b0da182414f6b791_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251816753/Put/seqid=0 2024-11-10T15:16:57,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251877448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251877445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251877461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251877461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742000_1176 (size=17284) 2024-11-10T15:16:57,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251877572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251877573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251877574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251877582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T15:16:57,592 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:57,596 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:16:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-10T15:16:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T15:16:57,609 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:16:57,611 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:16:57,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:16:57,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T15:16:57,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T15:16:57,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:57,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:57,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:57,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:57,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:57,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251877791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251877791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251877791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,800 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in 7c1cb26053f9cb07691f27c558e31c12 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251877791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251877792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:57,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:57,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:57,892 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:57,901 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110aa68cf23dc434794b0da182414f6b791_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110aa68cf23dc434794b0da182414f6b791_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:57,904 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/b4b09fed1d1e439e975a1e9e8f05c304, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:57,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/b4b09fed1d1e439e975a1e9e8f05c304 is 175, key is test_row_0/A:col10/1731251816753/Put/seqid=0 2024-11-10T15:16:57,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742001_1177 (size=48389) 2024-11-10T15:16:57,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T15:16:57,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:57,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:57,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:57,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:57,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:57,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:57,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T15:16:58,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T15:16:58,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:58,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:58,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251878111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:58,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:58,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:58,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:58,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251878111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:58,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:58,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:58,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251878111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:58,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:58,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251878111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:58,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T15:16:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T15:16:58,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:58,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:58,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:58,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,319 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/b4b09fed1d1e439e975a1e9e8f05c304 2024-11-10T15:16:58,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/cd815a5b31c141348fde00cbdf5d9eaf is 50, key is test_row_0/B:col10/1731251816753/Put/seqid=0 2024-11-10T15:16:58,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742002_1178 (size=12151) 2024-11-10T15:16:58,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T15:16:58,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:58,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:58,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:58,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T15:16:58,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:58,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:58,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:58,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251878623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:58,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:58,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:58,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:58,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251878631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:58,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251878632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:58,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251878632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:58,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T15:16:58,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:58,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:58,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:58,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T15:16:58,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/cd815a5b31c141348fde00cbdf5d9eaf 2024-11-10T15:16:58,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/e9673afec04f4c8d861131298b9b4835 is 50, key is test_row_0/C:col10/1731251816753/Put/seqid=0 2024-11-10T15:16:58,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742003_1179 (size=12151) 2024-11-10T15:16:58,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/e9673afec04f4c8d861131298b9b4835 2024-11-10T15:16:58,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/b4b09fed1d1e439e975a1e9e8f05c304 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/b4b09fed1d1e439e975a1e9e8f05c304 2024-11-10T15:16:58,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/b4b09fed1d1e439e975a1e9e8f05c304, entries=250, sequenceid=245, filesize=47.3 K 2024-11-10T15:16:58,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/cd815a5b31c141348fde00cbdf5d9eaf as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cd815a5b31c141348fde00cbdf5d9eaf 2024-11-10T15:16:58,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cd815a5b31c141348fde00cbdf5d9eaf, entries=150, sequenceid=245, filesize=11.9 K 2024-11-10T15:16:58,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/e9673afec04f4c8d861131298b9b4835 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/e9673afec04f4c8d861131298b9b4835 2024-11-10T15:16:58,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T15:16:58,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:58,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:58,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:58,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:58,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/e9673afec04f4c8d861131298b9b4835, entries=150, sequenceid=245, filesize=11.9 K 2024-11-10T15:16:58,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 7c1cb26053f9cb07691f27c558e31c12 in 1493ms, sequenceid=245, compaction requested=true 2024-11-10T15:16:58,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:58,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:16:58,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:58,908 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:16:58,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:16:58,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:58,908 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:16:58,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:16:58,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:58,912 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:16:58,912 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142148 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:16:58,912 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/B is initiating minor compaction (all files) 2024-11-10T15:16:58,912 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/A is initiating minor compaction (all files) 2024-11-10T15:16:58,912 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/A in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:58,912 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/70353ecee1c347119090ca62180f8152, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/508491b4d04e4705a3e2e9cfcac12460, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/40c2207c398b46ceb1face1fb047ddb9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/b4b09fed1d1e439e975a1e9e8f05c304] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=138.8 K 2024-11-10T15:16:58,912 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:58,912 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/B in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:58,912 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/70353ecee1c347119090ca62180f8152, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/508491b4d04e4705a3e2e9cfcac12460, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/40c2207c398b46ceb1face1fb047ddb9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/b4b09fed1d1e439e975a1e9e8f05c304] 2024-11-10T15:16:58,912 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/efa7ebb54d674fdaa739caba0ced0d7a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cc0761912f35488387cd878492fb785e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/dfa5af95010b421d899b3749770a9a47, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cd815a5b31c141348fde00cbdf5d9eaf] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=47.9 K 2024-11-10T15:16:58,914 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70353ecee1c347119090ca62180f8152, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731251814455 2024-11-10T15:16:58,916 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting efa7ebb54d674fdaa739caba0ced0d7a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731251814455 2024-11-10T15:16:58,917 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting cc0761912f35488387cd878492fb785e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731251815146 2024-11-10T15:16:58,917 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 508491b4d04e4705a3e2e9cfcac12460, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731251815146 2024-11-10T15:16:58,928 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting dfa5af95010b421d899b3749770a9a47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731251815526 2024-11-10T15:16:58,928 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40c2207c398b46ceb1face1fb047ddb9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731251815526 2024-11-10T15:16:58,932 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting cd815a5b31c141348fde00cbdf5d9eaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731251816746 2024-11-10T15:16:58,938 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4b09fed1d1e439e975a1e9e8f05c304, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731251816746 2024-11-10T15:16:58,968 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#B#compaction#147 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:58,968 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/db710ec591a245a5a81e8f94c42ce709 is 50, key is test_row_0/B:col10/1731251816753/Put/seqid=0 2024-11-10T15:16:58,981 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:59,003 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110833e328ad04c402ebcf49b63df934a29_7c1cb26053f9cb07691f27c558e31c12 store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:59,006 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110833e328ad04c402ebcf49b63df934a29_7c1cb26053f9cb07691f27c558e31c12, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:59,007 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110833e328ad04c402ebcf49b63df934a29_7c1cb26053f9cb07691f27c558e31c12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:59,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T15:16:59,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:59,015 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T15:16:59,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:16:59,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:59,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:16:59,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:59,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:16:59,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:59,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742004_1180 (size=12731) 2024-11-10T15:16:59,031 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/db710ec591a245a5a81e8f94c42ce709 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/db710ec591a245a5a81e8f94c42ce709 2024-11-10T15:16:59,042 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/B of 7c1cb26053f9cb07691f27c558e31c12 into db710ec591a245a5a81e8f94c42ce709(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:59,042 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:59,042 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/B, priority=12, startTime=1731251818908; duration=0sec 2024-11-10T15:16:59,042 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:16:59,042 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:B 2024-11-10T15:16:59,042 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:16:59,046 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:16:59,046 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/C is initiating minor compaction (all files) 2024-11-10T15:16:59,051 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/C in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:59,051 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c0fa1d359e3749cf9a2b61e25a9ddc42, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/36f901fde6e846e3af65fec7b1fdf02a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/1ca4b9559e6548e5bc3fc6166657b21b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/e9673afec04f4c8d861131298b9b4835] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=47.9 K 2024-11-10T15:16:59,053 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting c0fa1d359e3749cf9a2b61e25a9ddc42, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731251814455 2024-11-10T15:16:59,054 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 36f901fde6e846e3af65fec7b1fdf02a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731251815146 2024-11-10T15:16:59,054 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ca4b9559e6548e5bc3fc6166657b21b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731251815526 2024-11-10T15:16:59,059 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting e9673afec04f4c8d861131298b9b4835, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731251816746 2024-11-10T15:16:59,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742005_1181 (size=4469) 2024-11-10T15:16:59,079 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#A#compaction#148 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:59,080 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/fd3d19fafd8c467391d5670bc1ef19d9 is 175, key is test_row_0/A:col10/1731251816753/Put/seqid=0 2024-11-10T15:16:59,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107d0b6c74010e43b3ae93c343f87b0224_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251817435/Put/seqid=0 2024-11-10T15:16:59,105 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:16:59,106 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/60081e71e4d445f88f1078ebbceac9d8 is 50, key is test_row_0/C:col10/1731251816753/Put/seqid=0 2024-11-10T15:16:59,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742007_1183 (size=12404) 2024-11-10T15:16:59,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742008_1184 (size=12731) 2024-11-10T15:16:59,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742006_1182 (size=31685) 2024-11-10T15:16:59,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:59,140 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107d0b6c74010e43b3ae93c343f87b0224_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107d0b6c74010e43b3ae93c343f87b0224_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:59,144 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/fd3d19fafd8c467391d5670bc1ef19d9 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/fd3d19fafd8c467391d5670bc1ef19d9 2024-11-10T15:16:59,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/2ab810755a174210bf64046886c9ba41, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:59,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/2ab810755a174210bf64046886c9ba41 is 175, key is test_row_0/A:col10/1731251817435/Put/seqid=0 2024-11-10T15:16:59,148 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/60081e71e4d445f88f1078ebbceac9d8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/60081e71e4d445f88f1078ebbceac9d8 2024-11-10T15:16:59,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742009_1185 (size=31205) 2024-11-10T15:16:59,156 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/2ab810755a174210bf64046886c9ba41 2024-11-10T15:16:59,160 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/A of 7c1cb26053f9cb07691f27c558e31c12 into fd3d19fafd8c467391d5670bc1ef19d9(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:59,160 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:59,160 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/A, priority=12, startTime=1731251818908; duration=0sec 2024-11-10T15:16:59,160 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:59,160 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:A 2024-11-10T15:16:59,161 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/C of 7c1cb26053f9cb07691f27c558e31c12 into 60081e71e4d445f88f1078ebbceac9d8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:16:59,161 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:59,161 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/C, priority=12, startTime=1731251818908; duration=0sec 2024-11-10T15:16:59,162 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:16:59,162 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:C 2024-11-10T15:16:59,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/7bb187b3f1204e769f166ad4d41ce978 is 50, key is test_row_0/B:col10/1731251817435/Put/seqid=0 2024-11-10T15:16:59,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742010_1186 (size=12251) 2024-11-10T15:16:59,587 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/7bb187b3f1204e769f166ad4d41ce978 2024-11-10T15:16:59,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d9165abc3048496c9e52fc6f77cb3b96 is 50, key is test_row_0/C:col10/1731251817435/Put/seqid=0 2024-11-10T15:16:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:59,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:59,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742011_1187 (size=12251) 2024-11-10T15:16:59,643 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d9165abc3048496c9e52fc6f77cb3b96 2024-11-10T15:16:59,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/2ab810755a174210bf64046886c9ba41 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2ab810755a174210bf64046886c9ba41 2024-11-10T15:16:59,670 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2ab810755a174210bf64046886c9ba41, entries=150, sequenceid=260, filesize=30.5 K 2024-11-10T15:16:59,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/7bb187b3f1204e769f166ad4d41ce978 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/7bb187b3f1204e769f166ad4d41ce978 2024-11-10T15:16:59,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:59,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251879667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:59,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:59,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251879673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:59,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:59,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251879673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:59,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:59,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:59,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251879675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:59,687 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/7bb187b3f1204e769f166ad4d41ce978, entries=150, sequenceid=260, filesize=12.0 K 2024-11-10T15:16:59,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d9165abc3048496c9e52fc6f77cb3b96 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d9165abc3048496c9e52fc6f77cb3b96 2024-11-10T15:16:59,702 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d9165abc3048496c9e52fc6f77cb3b96, entries=150, sequenceid=260, filesize=12.0 K 2024-11-10T15:16:59,705 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 7c1cb26053f9cb07691f27c558e31c12 in 689ms, sequenceid=260, compaction requested=false 2024-11-10T15:16:59,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:16:59,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:59,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-10T15:16:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-10T15:16:59,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-10T15:16:59,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0960 sec 2024-11-10T15:16:59,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.1190 sec 2024-11-10T15:16:59,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T15:16:59,756 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:16:59,759 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:16:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-10T15:16:59,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T15:16:59,762 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:16:59,763 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:16:59,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:16:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:59,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T15:16:59,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:16:59,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:59,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:16:59,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:59,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:16:59,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:16:59,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105498c502ce2e4609a409e520b968880f_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251819782/Put/seqid=0 2024-11-10T15:16:59,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742012_1188 (size=12454) 2024-11-10T15:16:59,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251879804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:59,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:59,814 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:16:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251879801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:59,819 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105498c502ce2e4609a409e520b968880f_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105498c502ce2e4609a409e520b968880f_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:16:59,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251879812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:59,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:59,820 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/53553646695141adb0731c26cbcf7fc1, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:16:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251879813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:59,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/53553646695141adb0731c26cbcf7fc1 is 175, key is test_row_0/A:col10/1731251819782/Put/seqid=0 2024-11-10T15:16:59,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:59,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251879821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:59,824 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in 7c1cb26053f9cb07691f27c558e31c12 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:16:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:59,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742013_1189 (size=31255) 2024-11-10T15:16:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T15:16:59,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T15:16:59,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:59,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:16:59,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:16:59,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:59,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:16:59,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251879922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:59,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251879922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:59,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251879931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:59,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:16:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251879931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:16:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:16:59,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:16:59,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:00,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T15:17:00,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:00,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:00,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:00,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:00,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T15:17:00,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251880131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:00,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251880131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:00,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251880141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251880141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:00,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T15:17:00,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:00,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:00,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:00,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:00,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:00,256 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/53553646695141adb0731c26cbcf7fc1 2024-11-10T15:17:00,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/d4ceb14a50ac40b087f0914cb0e40a42 is 50, key is test_row_0/B:col10/1731251819782/Put/seqid=0 2024-11-10T15:17:00,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742014_1190 (size=12301) 2024-11-10T15:17:00,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/d4ceb14a50ac40b087f0914cb0e40a42 2024-11-10T15:17:00,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/8543997a132345ae9d8b0988d0ef9261 is 50, key is test_row_0/C:col10/1731251819782/Put/seqid=0 2024-11-10T15:17:00,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742015_1191 (size=12301) 2024-11-10T15:17:00,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T15:17:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:00,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T15:17:00,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251880441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251880441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:00,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251880451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:00,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251880451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:00,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T15:17:00,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:00,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:00,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:00,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:00,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T15:17:00,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:00,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:00,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:00,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:00,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:00,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/8543997a132345ae9d8b0988d0ef9261 2024-11-10T15:17:00,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/53553646695141adb0731c26cbcf7fc1 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/53553646695141adb0731c26cbcf7fc1 2024-11-10T15:17:00,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/53553646695141adb0731c26cbcf7fc1, entries=150, sequenceid=281, filesize=30.5 K 2024-11-10T15:17:00,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/d4ceb14a50ac40b087f0914cb0e40a42 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/d4ceb14a50ac40b087f0914cb0e40a42 2024-11-10T15:17:00,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/d4ceb14a50ac40b087f0914cb0e40a42, entries=150, sequenceid=281, filesize=12.0 K 2024-11-10T15:17:00,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/8543997a132345ae9d8b0988d0ef9261 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/8543997a132345ae9d8b0988d0ef9261 2024-11-10T15:17:00,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/8543997a132345ae9d8b0988d0ef9261, entries=150, sequenceid=281, filesize=12.0 K 2024-11-10T15:17:00,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 7c1cb26053f9cb07691f27c558e31c12 in 942ms, sequenceid=281, compaction requested=true 2024-11-10T15:17:00,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:00,727 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:00,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:17:00,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:00,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:17:00,730 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:00,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:00,730 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94145 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:00,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:17:00,730 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/A is initiating minor compaction (all files) 2024-11-10T15:17:00,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:00,731 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/A in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:00,731 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/fd3d19fafd8c467391d5670bc1ef19d9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2ab810755a174210bf64046886c9ba41, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/53553646695141adb0731c26cbcf7fc1] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=91.9 K 2024-11-10T15:17:00,731 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:00,731 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/fd3d19fafd8c467391d5670bc1ef19d9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2ab810755a174210bf64046886c9ba41, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/53553646695141adb0731c26cbcf7fc1] 2024-11-10T15:17:00,731 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:00,732 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/B is initiating minor compaction (all files) 2024-11-10T15:17:00,732 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/B in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:00,732 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/db710ec591a245a5a81e8f94c42ce709, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/7bb187b3f1204e769f166ad4d41ce978, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/d4ceb14a50ac40b087f0914cb0e40a42] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=36.4 K 2024-11-10T15:17:00,732 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting fd3d19fafd8c467391d5670bc1ef19d9, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731251816746 2024-11-10T15:17:00,733 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ab810755a174210bf64046886c9ba41, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731251817435 2024-11-10T15:17:00,733 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting db710ec591a245a5a81e8f94c42ce709, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731251816746 2024-11-10T15:17:00,733 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bb187b3f1204e769f166ad4d41ce978, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731251817435 2024-11-10T15:17:00,733 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 53553646695141adb0731c26cbcf7fc1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731251819665 2024-11-10T15:17:00,740 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4ceb14a50ac40b087f0914cb0e40a42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731251819665 2024-11-10T15:17:00,760 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:00,774 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#B#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:00,775 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/26d5814e8c0149a2a9461c2141486568 is 50, key is test_row_0/B:col10/1731251819782/Put/seqid=0 2024-11-10T15:17:00,789 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110795ffdaa03ba4bebbc704c885fc8162f_7c1cb26053f9cb07691f27c558e31c12 store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:00,791 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110795ffdaa03ba4bebbc704c885fc8162f_7c1cb26053f9cb07691f27c558e31c12, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:00,791 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110795ffdaa03ba4bebbc704c885fc8162f_7c1cb26053f9cb07691f27c558e31c12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:00,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742017_1193 (size=4469) 2024-11-10T15:17:00,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742016_1192 (size=12983) 2024-11-10T15:17:00,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T15:17:00,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:00,843 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T15:17:00,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:17:00,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:00,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:17:00,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:00,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:17:00,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:00,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111021a5294f9138443d943915edbd7a1ef1_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251819811/Put/seqid=0 2024-11-10T15:17:00,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742018_1194 (size=12454) 2024-11-10T15:17:00,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:00,878 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111021a5294f9138443d943915edbd7a1ef1_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111021a5294f9138443d943915edbd7a1ef1_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:00,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/cd9c44793b1c49d7bde737bdf416fe31, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:00,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/cd9c44793b1c49d7bde737bdf416fe31 is 175, key is test_row_0/A:col10/1731251819811/Put/seqid=0 2024-11-10T15:17:00,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742019_1195 (size=31255) 2024-11-10T15:17:00,887 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/cd9c44793b1c49d7bde737bdf416fe31 2024-11-10T15:17:00,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/6ea459440feb46d2879a0e7efef3ece9 is 50, key is test_row_0/B:col10/1731251819811/Put/seqid=0 2024-11-10T15:17:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T15:17:00,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742020_1196 (size=12301) 2024-11-10T15:17:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:00,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:00,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251880973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:00,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251880974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251880975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:00,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251880976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:00,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:00,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:01,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251881091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:01,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:01,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251881092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:01,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:01,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251881096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251881096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:01,223 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#A#compaction#156 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:01,223 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/2e7f71a1cedb4dbdb75cefafd762ff61 is 175, key is test_row_0/A:col10/1731251819782/Put/seqid=0 2024-11-10T15:17:01,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742021_1197 (size=31937) 2024-11-10T15:17:01,230 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/26d5814e8c0149a2a9461c2141486568 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/26d5814e8c0149a2a9461c2141486568 2024-11-10T15:17:01,239 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/B of 7c1cb26053f9cb07691f27c558e31c12 into 26d5814e8c0149a2a9461c2141486568(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:01,239 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:01,239 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/B, priority=13, startTime=1731251820730; duration=0sec 2024-11-10T15:17:01,239 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:01,239 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:B 2024-11-10T15:17:01,239 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:01,241 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:01,241 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/C is initiating minor compaction (all files) 2024-11-10T15:17:01,241 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/C in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:01,241 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/60081e71e4d445f88f1078ebbceac9d8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d9165abc3048496c9e52fc6f77cb3b96, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/8543997a132345ae9d8b0988d0ef9261] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=36.4 K 2024-11-10T15:17:01,242 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60081e71e4d445f88f1078ebbceac9d8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731251816746 2024-11-10T15:17:01,243 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9165abc3048496c9e52fc6f77cb3b96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731251817435 2024-11-10T15:17:01,243 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8543997a132345ae9d8b0988d0ef9261, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731251819665 2024-11-10T15:17:01,278 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#C#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:01,279 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/94df78e65e554d3c8b921a61f75cb5d7 is 50, key is test_row_0/C:col10/1731251819782/Put/seqid=0 2024-11-10T15:17:01,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742022_1198 (size=12983) 2024-11-10T15:17:01,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251881302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:01,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:01,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251881302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:01,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:01,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:01,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251881302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:01,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:01,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:01,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251881302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:01,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:01,320 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/6ea459440feb46d2879a0e7efef3ece9 2024-11-10T15:17:01,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/4bca5d496d4e4a399a73bea7a2fc9b95 is 50, key is test_row_0/C:col10/1731251819811/Put/seqid=0 2024-11-10T15:17:01,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742023_1199 (size=12301) 2024-11-10T15:17:01,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251881611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:01,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:01,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251881612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:01,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251881612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:01,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251881612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:01,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:01,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:01,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:01,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:01,636 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/2e7f71a1cedb4dbdb75cefafd762ff61 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2e7f71a1cedb4dbdb75cefafd762ff61 2024-11-10T15:17:01,645 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/A of 7c1cb26053f9cb07691f27c558e31c12 into 2e7f71a1cedb4dbdb75cefafd762ff61(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:01,645 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:01,645 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/A, priority=13, startTime=1731251820726; duration=0sec 2024-11-10T15:17:01,646 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:01,646 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:A 2024-11-10T15:17:01,711 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/94df78e65e554d3c8b921a61f75cb5d7 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/94df78e65e554d3c8b921a61f75cb5d7 2024-11-10T15:17:01,730 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/C of 7c1cb26053f9cb07691f27c558e31c12 into 94df78e65e554d3c8b921a61f75cb5d7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:01,730 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:01,730 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/C, priority=13, startTime=1731251820730; duration=0sec 2024-11-10T15:17:01,731 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:01,731 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:C 2024-11-10T15:17:01,747 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/4bca5d496d4e4a399a73bea7a2fc9b95 2024-11-10T15:17:01,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/cd9c44793b1c49d7bde737bdf416fe31 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/cd9c44793b1c49d7bde737bdf416fe31 2024-11-10T15:17:01,773 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/cd9c44793b1c49d7bde737bdf416fe31, entries=150, sequenceid=295, filesize=30.5 K 2024-11-10T15:17:01,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/6ea459440feb46d2879a0e7efef3ece9 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/6ea459440feb46d2879a0e7efef3ece9 2024-11-10T15:17:01,784 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/6ea459440feb46d2879a0e7efef3ece9, entries=150, sequenceid=295, filesize=12.0 K 2024-11-10T15:17:01,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/4bca5d496d4e4a399a73bea7a2fc9b95 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/4bca5d496d4e4a399a73bea7a2fc9b95 2024-11-10T15:17:01,795 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/4bca5d496d4e4a399a73bea7a2fc9b95, entries=150, sequenceid=295, filesize=12.0 K 2024-11-10T15:17:01,796 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 7c1cb26053f9cb07691f27c558e31c12 in 953ms, sequenceid=295, compaction requested=false 2024-11-10T15:17:01,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:01,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:01,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-10T15:17:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-10T15:17:01,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-10T15:17:01,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0350 sec 2024-11-10T15:17:01,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.0440 sec 2024-11-10T15:17:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T15:17:01,912 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:01,912 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:17:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-10T15:17:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T15:17:01,916 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:17:01,917 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:17:01,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:17:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T15:17:02,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T15:17:02,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:02,072 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T15:17:02,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:17:02,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:02,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:17:02,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:02,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:17:02,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:02,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111061294b6e496b4912bbfdf6d0d75e9c90_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251820971/Put/seqid=0 2024-11-10T15:17:02,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742024_1200 (size=12454) 2024-11-10T15:17:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:02,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:02,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251882144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251882146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251882154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251882159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T15:17:02,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251882262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251882262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251882271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251882271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251882472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251882472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251882481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251882481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:02,494 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111061294b6e496b4912bbfdf6d0d75e9c90_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111061294b6e496b4912bbfdf6d0d75e9c90_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:02,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/1fb0c7c41dd748168dbbd13c68532aa3, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:02,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/1fb0c7c41dd748168dbbd13c68532aa3 is 175, key is test_row_0/A:col10/1731251820971/Put/seqid=0 2024-11-10T15:17:02,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742025_1201 (size=31255) 2024-11-10T15:17:02,500 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/1fb0c7c41dd748168dbbd13c68532aa3 2024-11-10T15:17:02,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/30fd456bc30c45d6b5b7950641c74576 is 50, key is test_row_0/B:col10/1731251820971/Put/seqid=0 2024-11-10T15:17:02,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742026_1202 (size=12301) 2024-11-10T15:17:02,516 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/30fd456bc30c45d6b5b7950641c74576 2024-11-10T15:17:02,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d2853ef679af4b40971b6e432c7d8511 is 50, key is test_row_0/C:col10/1731251820971/Put/seqid=0 2024-11-10T15:17:02,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T15:17:02,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742027_1203 (size=12301) 2024-11-10T15:17:02,562 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d2853ef679af4b40971b6e432c7d8511 2024-11-10T15:17:02,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/1fb0c7c41dd748168dbbd13c68532aa3 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1fb0c7c41dd748168dbbd13c68532aa3 2024-11-10T15:17:02,575 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1fb0c7c41dd748168dbbd13c68532aa3, entries=150, sequenceid=316, filesize=30.5 K 2024-11-10T15:17:02,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/30fd456bc30c45d6b5b7950641c74576 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/30fd456bc30c45d6b5b7950641c74576 2024-11-10T15:17:02,580 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/30fd456bc30c45d6b5b7950641c74576, entries=150, sequenceid=316, filesize=12.0 K 2024-11-10T15:17:02,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d2853ef679af4b40971b6e432c7d8511 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d2853ef679af4b40971b6e432c7d8511 2024-11-10T15:17:02,587 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d2853ef679af4b40971b6e432c7d8511, entries=150, sequenceid=316, filesize=12.0 K 2024-11-10T15:17:02,588 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 7c1cb26053f9cb07691f27c558e31c12 in 516ms, sequenceid=316, compaction requested=true 2024-11-10T15:17:02,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:02,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:02,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-10T15:17:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-10T15:17:02,593 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-10T15:17:02,593 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 674 msec 2024-11-10T15:17:02,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 681 msec 2024-11-10T15:17:02,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:02,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T15:17:02,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:17:02,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:02,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:17:02,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:02,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:17:02,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:02,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111011b310eccf7b42a98533ba0303c7ad99_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251822143/Put/seqid=0 2024-11-10T15:17:02,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742028_1204 (size=14994) 2024-11-10T15:17:02,804 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:02,808 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111011b310eccf7b42a98533ba0303c7ad99_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111011b310eccf7b42a98533ba0303c7ad99_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:02,810 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/01ba6d70be5a4216b9ca88c5a092e9a7, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:02,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/01ba6d70be5a4216b9ca88c5a092e9a7 is 175, key is test_row_0/A:col10/1731251822143/Put/seqid=0 2024-11-10T15:17:02,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742029_1205 (size=39949) 2024-11-10T15:17:02,825 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/01ba6d70be5a4216b9ca88c5a092e9a7 2024-11-10T15:17:02,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251882823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251882823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251882825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251882832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/f83c790fe38741b38d268e6e7f7ab467 is 50, key is test_row_0/B:col10/1731251822143/Put/seqid=0 2024-11-10T15:17:02,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742030_1206 (size=12301) 2024-11-10T15:17:02,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/f83c790fe38741b38d268e6e7f7ab467 2024-11-10T15:17:02,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/0ef13013fc334e949b7263a0d29a926b is 50, key is test_row_0/C:col10/1731251822143/Put/seqid=0 2024-11-10T15:17:02,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742031_1207 (size=12301) 2024-11-10T15:17:02,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251882942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251882942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251882943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:02,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251882944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:02,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:02,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T15:17:03,052 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:03,053 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:17:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-10T15:17:03,064 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:17:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T15:17:03,065 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:17:03,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:17:03,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251883151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251883152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251883152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251883152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T15:17:03,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T15:17:03,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:03,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:03,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:03,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:03,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:03,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/0ef13013fc334e949b7263a0d29a926b 2024-11-10T15:17:03,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/01ba6d70be5a4216b9ca88c5a092e9a7 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/01ba6d70be5a4216b9ca88c5a092e9a7 2024-11-10T15:17:03,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/01ba6d70be5a4216b9ca88c5a092e9a7, entries=200, sequenceid=332, filesize=39.0 K 2024-11-10T15:17:03,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/f83c790fe38741b38d268e6e7f7ab467 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/f83c790fe38741b38d268e6e7f7ab467 2024-11-10T15:17:03,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/f83c790fe38741b38d268e6e7f7ab467, entries=150, sequenceid=332, filesize=12.0 K 2024-11-10T15:17:03,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/0ef13013fc334e949b7263a0d29a926b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/0ef13013fc334e949b7263a0d29a926b 2024-11-10T15:17:03,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/0ef13013fc334e949b7263a0d29a926b, entries=150, sequenceid=332, filesize=12.0 K 2024-11-10T15:17:03,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 7c1cb26053f9cb07691f27c558e31c12 in 526ms, sequenceid=332, compaction requested=true 2024-11-10T15:17:03,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:03,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:17:03,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:03,311 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:17:03,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:17:03,311 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:17:03,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:03,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:17:03,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:03,314 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:17:03,314 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:17:03,314 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/B is initiating minor compaction (all files) 2024-11-10T15:17:03,314 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/A is initiating minor compaction (all files) 2024-11-10T15:17:03,314 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/A in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:03,314 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/B in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:03,314 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2e7f71a1cedb4dbdb75cefafd762ff61, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/cd9c44793b1c49d7bde737bdf416fe31, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1fb0c7c41dd748168dbbd13c68532aa3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/01ba6d70be5a4216b9ca88c5a092e9a7] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=131.2 K 2024-11-10T15:17:03,314 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/26d5814e8c0149a2a9461c2141486568, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/6ea459440feb46d2879a0e7efef3ece9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/30fd456bc30c45d6b5b7950641c74576, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/f83c790fe38741b38d268e6e7f7ab467] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=48.7 K 2024-11-10T15:17:03,314 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:03,314 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2e7f71a1cedb4dbdb75cefafd762ff61, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/cd9c44793b1c49d7bde737bdf416fe31, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1fb0c7c41dd748168dbbd13c68532aa3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/01ba6d70be5a4216b9ca88c5a092e9a7] 2024-11-10T15:17:03,314 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e7f71a1cedb4dbdb75cefafd762ff61, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731251819665 2024-11-10T15:17:03,314 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 26d5814e8c0149a2a9461c2141486568, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731251819665 2024-11-10T15:17:03,315 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd9c44793b1c49d7bde737bdf416fe31, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731251819801 2024-11-10T15:17:03,315 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ea459440feb46d2879a0e7efef3ece9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731251819801 2024-11-10T15:17:03,315 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fb0c7c41dd748168dbbd13c68532aa3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731251820971 2024-11-10T15:17:03,315 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 30fd456bc30c45d6b5b7950641c74576, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731251820971 2024-11-10T15:17:03,315 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01ba6d70be5a4216b9ca88c5a092e9a7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731251822143 2024-11-10T15:17:03,315 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting f83c790fe38741b38d268e6e7f7ab467, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731251822143 2024-11-10T15:17:03,329 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:03,339 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#B#compaction#169 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:03,340 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/c921722cbe4e48fd8a98adc653ec69c0 is 50, key is test_row_0/B:col10/1731251822143/Put/seqid=0 2024-11-10T15:17:03,341 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411103b4fb73e18c548399783ab16a5aeedc7_7c1cb26053f9cb07691f27c558e31c12 store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:03,344 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411103b4fb73e18c548399783ab16a5aeedc7_7c1cb26053f9cb07691f27c558e31c12, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:03,344 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103b4fb73e18c548399783ab16a5aeedc7_7c1cb26053f9cb07691f27c558e31c12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:03,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742032_1208 (size=13119) 2024-11-10T15:17:03,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742033_1209 (size=4469) 2024-11-10T15:17:03,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T15:17:03,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:03,373 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T15:17:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:17:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:17:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:17:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T15:17:03,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111016ebf519ac4d406cb8647b094fe39595_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251822811/Put/seqid=0 2024-11-10T15:17:03,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742034_1210 (size=12454) 2024-11-10T15:17:03,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:03,446 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111016ebf519ac4d406cb8647b094fe39595_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111016ebf519ac4d406cb8647b094fe39595_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:03,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/41a5f7614a6a46708938041af00ff295, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:03,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/41a5f7614a6a46708938041af00ff295 is 175, key is test_row_0/A:col10/1731251822811/Put/seqid=0 2024-11-10T15:17:03,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:03,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742035_1211 (size=31255) 2024-11-10T15:17:03,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251883493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251883502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251883504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251883505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251883611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251883621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251883621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251883632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T15:17:03,768 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/c921722cbe4e48fd8a98adc653ec69c0 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/c921722cbe4e48fd8a98adc653ec69c0 2024-11-10T15:17:03,774 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#A#compaction#168 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:03,775 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/1d784ebdd9b9480b88a0287d6be0aca5 is 175, key is test_row_0/A:col10/1731251822143/Put/seqid=0 2024-11-10T15:17:03,777 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/B of 7c1cb26053f9cb07691f27c558e31c12 into c921722cbe4e48fd8a98adc653ec69c0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:03,778 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:03,784 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/B, priority=12, startTime=1731251823311; duration=0sec 2024-11-10T15:17:03,784 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:03,784 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:B 2024-11-10T15:17:03,784 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:17:03,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742036_1212 (size=32073) 2024-11-10T15:17:03,790 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:17:03,790 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/C is initiating minor compaction (all files) 2024-11-10T15:17:03,790 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/C in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:03,790 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/94df78e65e554d3c8b921a61f75cb5d7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/4bca5d496d4e4a399a73bea7a2fc9b95, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d2853ef679af4b40971b6e432c7d8511, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/0ef13013fc334e949b7263a0d29a926b] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=48.7 K 2024-11-10T15:17:03,790 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 94df78e65e554d3c8b921a61f75cb5d7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731251819665 2024-11-10T15:17:03,791 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bca5d496d4e4a399a73bea7a2fc9b95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731251819801 2024-11-10T15:17:03,791 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting d2853ef679af4b40971b6e432c7d8511, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731251820971 2024-11-10T15:17:03,793 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ef13013fc334e949b7263a0d29a926b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731251822143 2024-11-10T15:17:03,819 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#C#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:03,820 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/a319f8c3e7c84b5a993e7a3a14a0c5fc is 50, key is test_row_0/C:col10/1731251822143/Put/seqid=0 2024-11-10T15:17:03,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251883822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251883831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251883831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251883842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742037_1213 (size=13119) 2024-11-10T15:17:03,872 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/a319f8c3e7c84b5a993e7a3a14a0c5fc as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a319f8c3e7c84b5a993e7a3a14a0c5fc 2024-11-10T15:17:03,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55390 deadline: 1731251883871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:03,873 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in 7c1cb26053f9cb07691f27c558e31c12 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8307 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:03,882 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/C of 7c1cb26053f9cb07691f27c558e31c12 into a319f8c3e7c84b5a993e7a3a14a0c5fc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:03,882 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:03,882 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/C, priority=12, startTime=1731251823311; duration=0sec 2024-11-10T15:17:03,882 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:03,882 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:C 2024-11-10T15:17:03,901 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/41a5f7614a6a46708938041af00ff295 2024-11-10T15:17:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/277015bfc2964087a07dc1fb8fe05975 is 50, key is test_row_0/B:col10/1731251822811/Put/seqid=0 2024-11-10T15:17:03,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742038_1214 (size=12301) 2024-11-10T15:17:03,924 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/277015bfc2964087a07dc1fb8fe05975 2024-11-10T15:17:03,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d189ba5b37b642dbb85b1b84e7b1836d is 50, key is test_row_0/C:col10/1731251822811/Put/seqid=0 2024-11-10T15:17:03,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742039_1215 (size=12301) 2024-11-10T15:17:04,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251884131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:04,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251884142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:04,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251884151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251884151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:04,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T15:17:04,201 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/1d784ebdd9b9480b88a0287d6be0aca5 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1d784ebdd9b9480b88a0287d6be0aca5 2024-11-10T15:17:04,219 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/A of 7c1cb26053f9cb07691f27c558e31c12 into 1d784ebdd9b9480b88a0287d6be0aca5(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:04,219 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:04,219 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/A, priority=12, startTime=1731251823310; duration=0sec 2024-11-10T15:17:04,219 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:04,219 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:A 2024-11-10T15:17:04,345 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d189ba5b37b642dbb85b1b84e7b1836d 2024-11-10T15:17:04,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/41a5f7614a6a46708938041af00ff295 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/41a5f7614a6a46708938041af00ff295 2024-11-10T15:17:04,399 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/41a5f7614a6a46708938041af00ff295, entries=150, sequenceid=349, filesize=30.5 K 2024-11-10T15:17:04,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/277015bfc2964087a07dc1fb8fe05975 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/277015bfc2964087a07dc1fb8fe05975 2024-11-10T15:17:04,407 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/277015bfc2964087a07dc1fb8fe05975, entries=150, sequenceid=349, filesize=12.0 K 2024-11-10T15:17:04,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d189ba5b37b642dbb85b1b84e7b1836d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d189ba5b37b642dbb85b1b84e7b1836d 2024-11-10T15:17:04,425 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d189ba5b37b642dbb85b1b84e7b1836d, entries=150, sequenceid=349, filesize=12.0 K 2024-11-10T15:17:04,426 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 7c1cb26053f9cb07691f27c558e31c12 in 1053ms, sequenceid=349, compaction requested=false 2024-11-10T15:17:04,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:04,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:04,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-10T15:17:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-10T15:17:04,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-10T15:17:04,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3750 sec 2024-11-10T15:17:04,448 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.3910 sec 2024-11-10T15:17:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:04,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T15:17:04,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:17:04,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:04,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:17:04,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:04,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:17:04,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:04,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111051068fd7900f484ab9cc43b576631154_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251823481/Put/seqid=0 2024-11-10T15:17:04,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251884685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:04,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:04,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251884686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:04,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:04,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251884689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:04,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251884689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:04,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:04,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:04,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742040_1216 (size=14994) 2024-11-10T15:17:04,711 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:04,716 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111051068fd7900f484ab9cc43b576631154_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111051068fd7900f484ab9cc43b576631154_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:04,717 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/6cbfc44744c54f6db6f359ae10d675f2, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:04,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/6cbfc44744c54f6db6f359ae10d675f2 is 175, key is test_row_0/A:col10/1731251823481/Put/seqid=0 2024-11-10T15:17:04,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742041_1217 (size=39949) 2024-11-10T15:17:04,737 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/6cbfc44744c54f6db6f359ae10d675f2 2024-11-10T15:17:04,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/8fdacaf049e24494aafa580ae2d25d20 is 50, key is test_row_0/B:col10/1731251823481/Put/seqid=0 2024-11-10T15:17:04,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742042_1218 (size=12301) 2024-11-10T15:17:04,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:04,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251884791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:04,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:04,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251884801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:04,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:04,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251884812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:04,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:04,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251884812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:05,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251885001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:05,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:05,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251885012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:05,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251885021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:05,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:05,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251885024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:05,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/8fdacaf049e24494aafa580ae2d25d20 2024-11-10T15:17:05,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/69843d87e59c49478f38af829d5c55b8 is 50, key is test_row_0/C:col10/1731251823481/Put/seqid=0 2024-11-10T15:17:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T15:17:05,212 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:05,213 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:17:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-10T15:17:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T15:17:05,215 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:17:05,216 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:17:05,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:17:05,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742043_1219 (size=12301) 2024-11-10T15:17:05,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251885311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T15:17:05,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251885321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:05,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:05,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251885332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:05,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251885348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:05,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:05,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:05,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:05,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:05,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:05,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:05,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:05,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:05,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:05,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:05,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:05,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:05,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T15:17:05,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/69843d87e59c49478f38af829d5c55b8 2024-11-10T15:17:05,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/6cbfc44744c54f6db6f359ae10d675f2 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/6cbfc44744c54f6db6f359ae10d675f2 2024-11-10T15:17:05,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/6cbfc44744c54f6db6f359ae10d675f2, entries=200, sequenceid=368, filesize=39.0 K 2024-11-10T15:17:05,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/8fdacaf049e24494aafa580ae2d25d20 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/8fdacaf049e24494aafa580ae2d25d20 2024-11-10T15:17:05,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:05,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:05,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:05,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:05,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:05,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:05,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/8fdacaf049e24494aafa580ae2d25d20, entries=150, sequenceid=368, filesize=12.0 K 2024-11-10T15:17:05,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/69843d87e59c49478f38af829d5c55b8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/69843d87e59c49478f38af829d5c55b8 2024-11-10T15:17:05,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/69843d87e59c49478f38af829d5c55b8, entries=150, sequenceid=368, filesize=12.0 K 2024-11-10T15:17:05,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 7c1cb26053f9cb07691f27c558e31c12 in 1044ms, sequenceid=368, compaction requested=true 2024-11-10T15:17:05,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:05,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:17:05,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:05,693 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:05,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:17:05,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:05,693 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:05,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:17:05,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:05,694 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:05,694 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/A is initiating minor compaction (all files) 2024-11-10T15:17:05,694 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/A in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:05,694 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1d784ebdd9b9480b88a0287d6be0aca5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/41a5f7614a6a46708938041af00ff295, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/6cbfc44744c54f6db6f359ae10d675f2] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=100.9 K 2024-11-10T15:17:05,695 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:05,695 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1d784ebdd9b9480b88a0287d6be0aca5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/41a5f7614a6a46708938041af00ff295, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/6cbfc44744c54f6db6f359ae10d675f2] 2024-11-10T15:17:05,695 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:05,695 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d784ebdd9b9480b88a0287d6be0aca5, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731251822143 2024-11-10T15:17:05,695 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/B is initiating minor compaction (all files) 2024-11-10T15:17:05,695 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/B in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:05,695 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/c921722cbe4e48fd8a98adc653ec69c0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/277015bfc2964087a07dc1fb8fe05975, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/8fdacaf049e24494aafa580ae2d25d20] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=36.8 K 2024-11-10T15:17:05,695 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41a5f7614a6a46708938041af00ff295, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731251822811 2024-11-10T15:17:05,695 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting c921722cbe4e48fd8a98adc653ec69c0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731251822143 2024-11-10T15:17:05,696 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cbfc44744c54f6db6f359ae10d675f2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731251823481 2024-11-10T15:17:05,696 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 277015bfc2964087a07dc1fb8fe05975, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731251822811 2024-11-10T15:17:05,696 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fdacaf049e24494aafa580ae2d25d20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731251823481 2024-11-10T15:17:05,705 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:05,717 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#B#compaction#178 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:05,717 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/112513cf36524d8bb6070e6cfea98319 is 50, key is test_row_0/B:col10/1731251823481/Put/seqid=0 2024-11-10T15:17:05,718 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110fc6447e3705a4744975f9ff45d6fd366_7c1cb26053f9cb07691f27c558e31c12 store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:05,721 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110fc6447e3705a4744975f9ff45d6fd366_7c1cb26053f9cb07691f27c558e31c12, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:05,721 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fc6447e3705a4744975f9ff45d6fd366_7c1cb26053f9cb07691f27c558e31c12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:05,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742045_1221 (size=4469) 2024-11-10T15:17:05,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742044_1220 (size=13221) 2024-11-10T15:17:05,754 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/112513cf36524d8bb6070e6cfea98319 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/112513cf36524d8bb6070e6cfea98319 2024-11-10T15:17:05,765 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/B of 7c1cb26053f9cb07691f27c558e31c12 into 112513cf36524d8bb6070e6cfea98319(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:05,765 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:05,765 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/B, priority=13, startTime=1731251825693; duration=0sec 2024-11-10T15:17:05,766 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:05,766 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:B 2024-11-10T15:17:05,766 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:05,767 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:05,767 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/C is initiating minor compaction (all files) 2024-11-10T15:17:05,767 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/C in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:05,768 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a319f8c3e7c84b5a993e7a3a14a0c5fc, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d189ba5b37b642dbb85b1b84e7b1836d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/69843d87e59c49478f38af829d5c55b8] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=36.8 K 2024-11-10T15:17:05,768 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting a319f8c3e7c84b5a993e7a3a14a0c5fc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731251822143 2024-11-10T15:17:05,769 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting d189ba5b37b642dbb85b1b84e7b1836d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731251822811 2024-11-10T15:17:05,769 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 69843d87e59c49478f38af829d5c55b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731251823481 2024-11-10T15:17:05,788 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#C#compaction#179 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:05,789 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/556204548f2a497e8ff9ddc66dfee94e is 50, key is test_row_0/C:col10/1731251823481/Put/seqid=0 2024-11-10T15:17:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:05,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T15:17:05,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:17:05,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:05,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:17:05,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:05,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:17:05,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:05,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742046_1222 (size=13221) 2024-11-10T15:17:05,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:05,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:05,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:05,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:05,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:05,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T15:17:05,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ff4d22e3a6ba40cc8fc6fe8d9774a684_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251824684/Put/seqid=0 2024-11-10T15:17:05,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742047_1223 (size=14994) 2024-11-10T15:17:05,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251885894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:05,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251885897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:05,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251885898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:05,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251885899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:05,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:05,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:05,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:05,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:05,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:05,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:05,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251886012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251886012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251886012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251886012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,141 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#A#compaction#177 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:06,142 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/7c0a2bd708e24368a74ececfcefa5065 is 175, key is test_row_0/A:col10/1731251823481/Put/seqid=0 2024-11-10T15:17:06,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:06,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:06,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:06,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:06,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742048_1224 (size=32175) 2024-11-10T15:17:06,156 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/7c0a2bd708e24368a74ececfcefa5065 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/7c0a2bd708e24368a74ececfcefa5065 2024-11-10T15:17:06,161 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/A of 7c1cb26053f9cb07691f27c558e31c12 into 7c0a2bd708e24368a74ececfcefa5065(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:06,161 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:06,161 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/A, priority=13, startTime=1731251825693; duration=0sec 2024-11-10T15:17:06,161 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:06,161 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:A 2024-11-10T15:17:06,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251886221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251886221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251886221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251886221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,234 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/556204548f2a497e8ff9ddc66dfee94e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/556204548f2a497e8ff9ddc66dfee94e 2024-11-10T15:17:06,241 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/C of 7c1cb26053f9cb07691f27c558e31c12 into 556204548f2a497e8ff9ddc66dfee94e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:06,241 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:06,241 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/C, priority=13, startTime=1731251825693; duration=0sec 2024-11-10T15:17:06,241 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:06,241 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:C 2024-11-10T15:17:06,287 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:06,291 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ff4d22e3a6ba40cc8fc6fe8d9774a684_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ff4d22e3a6ba40cc8fc6fe8d9774a684_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:06,292 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/e698554e212e4565a449fdd1e889282d, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:06,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/e698554e212e4565a449fdd1e889282d is 175, key is test_row_0/A:col10/1731251824684/Put/seqid=0 2024-11-10T15:17:06,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:06,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:06,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:06,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:06,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742049_1225 (size=39949) 2024-11-10T15:17:06,311 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=387, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/e698554e212e4565a449fdd1e889282d 2024-11-10T15:17:06,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/bf75df38e65543eea0ab3140e8fa3025 is 50, key is test_row_0/B:col10/1731251824684/Put/seqid=0 2024-11-10T15:17:06,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742050_1226 (size=12301) 2024-11-10T15:17:06,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/bf75df38e65543eea0ab3140e8fa3025 2024-11-10T15:17:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T15:17:06,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/2093820aa2a84420a118359a7041d140 is 50, key is test_row_0/C:col10/1731251824684/Put/seqid=0 2024-11-10T15:17:06,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742051_1227 (size=12301) 2024-11-10T15:17:06,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/2093820aa2a84420a118359a7041d140 2024-11-10T15:17:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/e698554e212e4565a449fdd1e889282d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/e698554e212e4565a449fdd1e889282d 2024-11-10T15:17:06,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/e698554e212e4565a449fdd1e889282d, entries=200, sequenceid=387, filesize=39.0 K 2024-11-10T15:17:06,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/bf75df38e65543eea0ab3140e8fa3025 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/bf75df38e65543eea0ab3140e8fa3025 2024-11-10T15:17:06,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:06,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:06,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:06,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:06,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/bf75df38e65543eea0ab3140e8fa3025, entries=150, sequenceid=387, filesize=12.0 K 2024-11-10T15:17:06,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/2093820aa2a84420a118359a7041d140 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2093820aa2a84420a118359a7041d140 2024-11-10T15:17:06,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2093820aa2a84420a118359a7041d140, entries=150, sequenceid=387, filesize=12.0 K 2024-11-10T15:17:06,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 7c1cb26053f9cb07691f27c558e31c12 in 648ms, sequenceid=387, compaction requested=false 2024-11-10T15:17:06,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:06,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T15:17:06,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:17:06,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:06,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:17:06,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:06,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:17:06,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:06,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111019019805e14346b1a8a3a3e5d162bc51_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251825889/Put/seqid=0 2024-11-10T15:17:06,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742052_1228 (size=14994) 2024-11-10T15:17:06,550 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:06,554 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111019019805e14346b1a8a3a3e5d162bc51_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111019019805e14346b1a8a3a3e5d162bc51_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:06,555 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/2236541118cb4022b7d637fec4ca8fc5, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:06,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/2236541118cb4022b7d637fec4ca8fc5 is 175, key is test_row_0/A:col10/1731251825889/Put/seqid=0 2024-11-10T15:17:06,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251886554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251886557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251886557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251886558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742053_1229 (size=39949) 2024-11-10T15:17:06,571 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=404, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/2236541118cb4022b7d637fec4ca8fc5 2024-11-10T15:17:06,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/614d747e30c04802a486f1330511fc28 is 50, key is test_row_0/B:col10/1731251825889/Put/seqid=0 2024-11-10T15:17:06,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742054_1230 (size=12301) 2024-11-10T15:17:06,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/614d747e30c04802a486f1330511fc28 2024-11-10T15:17:06,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:06,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:06,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:06,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:06,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/a2b81ad4844748a5a301b1864984b838 is 50, key is test_row_0/C:col10/1731251825889/Put/seqid=0 2024-11-10T15:17:06,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742055_1231 (size=12301) 2024-11-10T15:17:06,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/a2b81ad4844748a5a301b1864984b838 2024-11-10T15:17:06,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/2236541118cb4022b7d637fec4ca8fc5 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2236541118cb4022b7d637fec4ca8fc5 2024-11-10T15:17:06,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2236541118cb4022b7d637fec4ca8fc5, entries=200, sequenceid=404, filesize=39.0 K 2024-11-10T15:17:06,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/614d747e30c04802a486f1330511fc28 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/614d747e30c04802a486f1330511fc28 2024-11-10T15:17:06,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/614d747e30c04802a486f1330511fc28, entries=150, sequenceid=404, filesize=12.0 K 2024-11-10T15:17:06,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/a2b81ad4844748a5a301b1864984b838 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a2b81ad4844748a5a301b1864984b838 2024-11-10T15:17:06,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a2b81ad4844748a5a301b1864984b838, entries=150, sequenceid=404, filesize=12.0 K 2024-11-10T15:17:06,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 7c1cb26053f9cb07691f27c558e31c12 in 122ms, sequenceid=404, compaction requested=true 2024-11-10T15:17:06,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:06,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:17:06,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:06,655 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:06,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:17:06,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:06,655 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:06,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:17:06,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:06,656 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 112073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:06,656 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/A is initiating minor compaction (all files) 2024-11-10T15:17:06,656 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/A in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:06,656 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/7c0a2bd708e24368a74ececfcefa5065, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/e698554e212e4565a449fdd1e889282d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2236541118cb4022b7d637fec4ca8fc5] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=109.4 K 2024-11-10T15:17:06,656 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:06,658 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/B is initiating minor compaction (all files) 2024-11-10T15:17:06,658 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:06,658 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/7c0a2bd708e24368a74ececfcefa5065, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/e698554e212e4565a449fdd1e889282d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2236541118cb4022b7d637fec4ca8fc5] 2024-11-10T15:17:06,658 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/B in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:06,658 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/112513cf36524d8bb6070e6cfea98319, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/bf75df38e65543eea0ab3140e8fa3025, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/614d747e30c04802a486f1330511fc28] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=36.9 K 2024-11-10T15:17:06,658 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c0a2bd708e24368a74ececfcefa5065, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731251823481 2024-11-10T15:17:06,658 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 112513cf36524d8bb6070e6cfea98319, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731251823481 2024-11-10T15:17:06,659 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e698554e212e4565a449fdd1e889282d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731251824678 2024-11-10T15:17:06,659 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting bf75df38e65543eea0ab3140e8fa3025, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731251824684 2024-11-10T15:17:06,659 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2236541118cb4022b7d637fec4ca8fc5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731251825856 2024-11-10T15:17:06,659 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 614d747e30c04802a486f1330511fc28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731251825889 2024-11-10T15:17:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:06,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T15:17:06,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:17:06,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:06,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:17:06,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:06,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:17:06,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:06,671 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:06,672 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#B#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:06,673 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/22a8453239e543beb12ccb749c365e70 is 50, key is test_row_0/B:col10/1731251825889/Put/seqid=0 2024-11-10T15:17:06,683 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110756d1520a1a6490fbab1afb9f6b98ee1_7c1cb26053f9cb07691f27c558e31c12 store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:06,686 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110756d1520a1a6490fbab1afb9f6b98ee1_7c1cb26053f9cb07691f27c558e31c12, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:06,686 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110756d1520a1a6490fbab1afb9f6b98ee1_7c1cb26053f9cb07691f27c558e31c12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:06,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107509ba075e3d496596344575fddf3bc3_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251826548/Put/seqid=0 2024-11-10T15:17:06,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742057_1233 (size=4469) 2024-11-10T15:17:06,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742056_1232 (size=13323) 2024-11-10T15:17:06,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251886687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251886688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251886688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742058_1234 (size=14994) 2024-11-10T15:17:06,701 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:06,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251886699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,708 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107509ba075e3d496596344575fddf3bc3_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107509ba075e3d496596344575fddf3bc3_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:06,710 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/83178f08f5df4b14b5ca62b0b714d3df, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:06,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/83178f08f5df4b14b5ca62b0b714d3df is 175, key is test_row_0/A:col10/1731251826548/Put/seqid=0 2024-11-10T15:17:06,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742059_1235 (size=39949) 2024-11-10T15:17:06,729 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=422, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/83178f08f5df4b14b5ca62b0b714d3df 2024-11-10T15:17:06,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/4d1293f7d638446ebe2719769accb97a is 50, key is test_row_0/B:col10/1731251826548/Put/seqid=0 2024-11-10T15:17:06,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742060_1236 (size=12301) 2024-11-10T15:17:06,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/4d1293f7d638446ebe2719769accb97a 2024-11-10T15:17:06,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:06,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:06,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:06,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:06,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/c53864e8e9ac4fbdb0f0c1f3234dbd92 is 50, key is test_row_0/C:col10/1731251826548/Put/seqid=0 2024-11-10T15:17:06,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742061_1237 (size=12301) 2024-11-10T15:17:06,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251886802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251886802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251886811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:06,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251886811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:06,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:06,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:06,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:06,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:06,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:06,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:06,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251887012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:07,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251887012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:07,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251887022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:07,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251887022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:07,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:07,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:07,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:07,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:07,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,096 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#A#compaction#186 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:07,097 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/269422976e93461fafce2fc4a19191f6 is 175, key is test_row_0/A:col10/1731251825889/Put/seqid=0 2024-11-10T15:17:07,109 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/22a8453239e543beb12ccb749c365e70 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/22a8453239e543beb12ccb749c365e70 2024-11-10T15:17:07,120 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/B of 7c1cb26053f9cb07691f27c558e31c12 into 22a8453239e543beb12ccb749c365e70(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:07,120 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:07,120 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/B, priority=13, startTime=1731251826655; duration=0sec 2024-11-10T15:17:07,120 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:07,120 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:B 2024-11-10T15:17:07,120 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:07,122 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:07,122 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/C is initiating minor compaction (all files) 2024-11-10T15:17:07,122 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/C in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:07,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742062_1238 (size=32277) 2024-11-10T15:17:07,123 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/556204548f2a497e8ff9ddc66dfee94e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2093820aa2a84420a118359a7041d140, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a2b81ad4844748a5a301b1864984b838] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=36.9 K 2024-11-10T15:17:07,123 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 556204548f2a497e8ff9ddc66dfee94e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731251823481 2024-11-10T15:17:07,126 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2093820aa2a84420a118359a7041d140, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731251824684 2024-11-10T15:17:07,127 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting a2b81ad4844748a5a301b1864984b838, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731251825889 2024-11-10T15:17:07,132 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/269422976e93461fafce2fc4a19191f6 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/269422976e93461fafce2fc4a19191f6 2024-11-10T15:17:07,147 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/A of 7c1cb26053f9cb07691f27c558e31c12 into 269422976e93461fafce2fc4a19191f6(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:07,147 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:07,147 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/A, priority=13, startTime=1731251826655; duration=0sec 2024-11-10T15:17:07,147 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:07,147 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:A 2024-11-10T15:17:07,172 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#C#compaction#191 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:07,172 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d3ae7201012044e189e8589b746fb159 is 50, key is test_row_0/C:col10/1731251825889/Put/seqid=0 2024-11-10T15:17:07,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742063_1239 (size=13323) 2024-11-10T15:17:07,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/c53864e8e9ac4fbdb0f0c1f3234dbd92 2024-11-10T15:17:07,186 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/d3ae7201012044e189e8589b746fb159 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d3ae7201012044e189e8589b746fb159 2024-11-10T15:17:07,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/83178f08f5df4b14b5ca62b0b714d3df as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/83178f08f5df4b14b5ca62b0b714d3df 2024-11-10T15:17:07,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/83178f08f5df4b14b5ca62b0b714d3df, entries=200, sequenceid=422, filesize=39.0 K 2024-11-10T15:17:07,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/4d1293f7d638446ebe2719769accb97a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/4d1293f7d638446ebe2719769accb97a 2024-11-10T15:17:07,206 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/C of 7c1cb26053f9cb07691f27c558e31c12 into d3ae7201012044e189e8589b746fb159(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:07,206 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:07,206 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/C, priority=13, startTime=1731251826655; duration=0sec 2024-11-10T15:17:07,206 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:07,206 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:C 2024-11-10T15:17:07,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/4d1293f7d638446ebe2719769accb97a, entries=150, sequenceid=422, filesize=12.0 K 2024-11-10T15:17:07,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/c53864e8e9ac4fbdb0f0c1f3234dbd92 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c53864e8e9ac4fbdb0f0c1f3234dbd92 2024-11-10T15:17:07,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:07,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:07,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:07,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:07,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c53864e8e9ac4fbdb0f0c1f3234dbd92, entries=150, sequenceid=422, filesize=12.0 K 2024-11-10T15:17:07,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 7c1cb26053f9cb07691f27c558e31c12 in 566ms, sequenceid=422, compaction requested=false 2024-11-10T15:17:07,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:07,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T15:17:07,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:17:07,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:07,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:17:07,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:07,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:17:07,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:07,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a40046b8cb7540ac919162cca91965e6_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251827328/Put/seqid=0 2024-11-10T15:17:07,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742064_1240 (size=14994) 2024-11-10T15:17:07,355 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:07,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251887351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:07,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:07,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251887352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:07,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:07,361 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a40046b8cb7540ac919162cca91965e6_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a40046b8cb7540ac919162cca91965e6_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:07,362 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/a45749c8c845450fbc9784aff4199eaa, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:07,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/a45749c8c845450fbc9784aff4199eaa is 175, key is test_row_0/A:col10/1731251827328/Put/seqid=0 2024-11-10T15:17:07,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251887354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:07,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251887356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T15:17:07,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:07,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:07,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:07,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:07,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742065_1241 (size=39949) 2024-11-10T15:17:07,393 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=440, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/a45749c8c845450fbc9784aff4199eaa 2024-11-10T15:17:07,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/b29dc6fa6a4540b3a9f4151f8746423e is 50, key is test_row_0/B:col10/1731251827328/Put/seqid=0 2024-11-10T15:17:07,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742066_1242 (size=12301) 2024-11-10T15:17:07,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/b29dc6fa6a4540b3a9f4151f8746423e 2024-11-10T15:17:07,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/4d92aae6dded4fa9a32ca5ae9727ff8a is 50, key is test_row_0/C:col10/1731251827328/Put/seqid=0 2024-11-10T15:17:07,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742067_1243 (size=12301) 2024-11-10T15:17:07,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251887462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:07,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251887462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:07,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251887471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251887471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:07,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:07,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:07,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:07,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:07,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251887672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:07,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251887672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:07,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251887682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251887682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:07,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:07,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:07,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:07,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:07,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/4d92aae6dded4fa9a32ca5ae9727ff8a 2024-11-10T15:17:07,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:07,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:07,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:07,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:07,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:07,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/a45749c8c845450fbc9784aff4199eaa as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a45749c8c845450fbc9784aff4199eaa 2024-11-10T15:17:07,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a45749c8c845450fbc9784aff4199eaa, entries=200, sequenceid=440, filesize=39.0 K 2024-11-10T15:17:07,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/b29dc6fa6a4540b3a9f4151f8746423e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/b29dc6fa6a4540b3a9f4151f8746423e 2024-11-10T15:17:07,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/b29dc6fa6a4540b3a9f4151f8746423e, entries=150, sequenceid=440, filesize=12.0 K 2024-11-10T15:17:07,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/4d92aae6dded4fa9a32ca5ae9727ff8a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/4d92aae6dded4fa9a32ca5ae9727ff8a 2024-11-10T15:17:07,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/4d92aae6dded4fa9a32ca5ae9727ff8a, entries=150, sequenceid=440, filesize=12.0 K 2024-11-10T15:17:07,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 7c1cb26053f9cb07691f27c558e31c12 in 557ms, sequenceid=440, compaction requested=true 2024-11-10T15:17:07,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:07,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:17:07,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:07,886 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:07,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:17:07,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:07,886 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:07,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c1cb26053f9cb07691f27c558e31c12:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:17:07,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:07,888 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:07,888 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/B is initiating minor compaction (all files) 2024-11-10T15:17:07,888 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/B in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:07,888 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/22a8453239e543beb12ccb749c365e70, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/4d1293f7d638446ebe2719769accb97a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/b29dc6fa6a4540b3a9f4151f8746423e] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=37.0 K 2024-11-10T15:17:07,888 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 112175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:07,888 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/A is initiating minor compaction (all files) 2024-11-10T15:17:07,888 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/A in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:07,888 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/269422976e93461fafce2fc4a19191f6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/83178f08f5df4b14b5ca62b0b714d3df, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a45749c8c845450fbc9784aff4199eaa] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=109.5 K 2024-11-10T15:17:07,888 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 22a8453239e543beb12ccb749c365e70, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731251825889 2024-11-10T15:17:07,888 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:07,888 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/269422976e93461fafce2fc4a19191f6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/83178f08f5df4b14b5ca62b0b714d3df, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a45749c8c845450fbc9784aff4199eaa] 2024-11-10T15:17:07,889 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d1293f7d638446ebe2719769accb97a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731251826548 2024-11-10T15:17:07,889 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting b29dc6fa6a4540b3a9f4151f8746423e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731251826685 2024-11-10T15:17:07,889 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 269422976e93461fafce2fc4a19191f6, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731251825889 2024-11-10T15:17:07,890 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83178f08f5df4b14b5ca62b0b714d3df, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731251826548 2024-11-10T15:17:07,891 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a45749c8c845450fbc9784aff4199eaa, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731251826685 2024-11-10T15:17:07,901 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:07,914 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:07,915 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/e211724178ad4390a79fdf55fd2fa20d is 50, key is test_row_0/B:col10/1731251827328/Put/seqid=0 2024-11-10T15:17:07,925 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110ee9a64c3fcd6400982f39428be89fba0_7c1cb26053f9cb07691f27c558e31c12 store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:07,927 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110ee9a64c3fcd6400982f39428be89fba0_7c1cb26053f9cb07691f27c558e31c12, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:07,927 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ee9a64c3fcd6400982f39428be89fba0_7c1cb26053f9cb07691f27c558e31c12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:07,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742068_1244 (size=13425) 2024-11-10T15:17:07,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742069_1245 (size=4469) 2024-11-10T15:17:07,983 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#A#compaction#195 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:07,984 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/24a879e5e74c44ab8365fb9753fdecec is 175, key is test_row_0/A:col10/1731251827328/Put/seqid=0 2024-11-10T15:17:07,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T15:17:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:07,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:17:07,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:07,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:17:07,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:07,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:17:07,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:08,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:08,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:08,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:08,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:08,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:08,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:08,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:08,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:08,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:08,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251888025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:08,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251888023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:08,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:08,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:08,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:08,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:08,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251888026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:08,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742070_1246 (size=32379) 2024-11-10T15:17:08,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105651d694fa324ad2bc24d25edd9fede3_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251827985/Put/seqid=0 2024-11-10T15:17:08,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:08,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251888034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:08,047 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/24a879e5e74c44ab8365fb9753fdecec as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/24a879e5e74c44ab8365fb9753fdecec 2024-11-10T15:17:08,054 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/A of 7c1cb26053f9cb07691f27c558e31c12 into 24a879e5e74c44ab8365fb9753fdecec(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:08,054 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:08,055 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/A, priority=13, startTime=1731251827886; duration=0sec 2024-11-10T15:17:08,055 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:08,055 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:A 2024-11-10T15:17:08,055 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:08,056 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:08,057 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): 7c1cb26053f9cb07691f27c558e31c12/C is initiating minor compaction (all files) 2024-11-10T15:17:08,057 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c1cb26053f9cb07691f27c558e31c12/C in TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:08,057 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d3ae7201012044e189e8589b746fb159, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c53864e8e9ac4fbdb0f0c1f3234dbd92, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/4d92aae6dded4fa9a32ca5ae9727ff8a] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp, totalSize=37.0 K 2024-11-10T15:17:08,058 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3ae7201012044e189e8589b746fb159, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731251825889 2024-11-10T15:17:08,058 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c53864e8e9ac4fbdb0f0c1f3234dbd92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731251826548 2024-11-10T15:17:08,058 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d92aae6dded4fa9a32ca5ae9727ff8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731251826685 2024-11-10T15:17:08,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742071_1247 (size=17534) 2024-11-10T15:17:08,090 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:08,095 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105651d694fa324ad2bc24d25edd9fede3_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105651d694fa324ad2bc24d25edd9fede3_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:08,097 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/dfe3a8bde81f4d218db4436bf9417800, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:08,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/dfe3a8bde81f4d218db4436bf9417800 is 175, key is test_row_0/A:col10/1731251827985/Put/seqid=0 2024-11-10T15:17:08,119 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c1cb26053f9cb07691f27c558e31c12#C#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:08,119 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/aeaecccc8f584c1fae8e1aee98ec0916 is 50, key is test_row_0/C:col10/1731251827328/Put/seqid=0 2024-11-10T15:17:08,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742072_1248 (size=48639) 2024-11-10T15:17:08,138 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=459, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/dfe3a8bde81f4d218db4436bf9417800 2024-11-10T15:17:08,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:08,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55364 deadline: 1731251888142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55370 deadline: 1731251888142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:08,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:08,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55350 deadline: 1731251888142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:08,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:08,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55384 deadline: 1731251888153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:08,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:08,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:08,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:08,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:08,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:08,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:08,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:08,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:08,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:08,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:08,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/4c2475b3e7334bb784c7cbb487a0e122 is 50, key is test_row_0/B:col10/1731251827985/Put/seqid=0 2024-11-10T15:17:08,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742073_1249 (size=13425) 2024-11-10T15:17:08,204 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/aeaecccc8f584c1fae8e1aee98ec0916 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/aeaecccc8f584c1fae8e1aee98ec0916 2024-11-10T15:17:08,211 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/C of 7c1cb26053f9cb07691f27c558e31c12 into aeaecccc8f584c1fae8e1aee98ec0916(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:08,211 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:08,211 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/C, priority=13, startTime=1731251827886; duration=0sec 2024-11-10T15:17:08,211 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:08,211 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:C 2024-11-10T15:17:08,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742074_1250 (size=12301) 2024-11-10T15:17:08,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/4c2475b3e7334bb784c7cbb487a0e122 2024-11-10T15:17:08,226 INFO [Thread-584 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-584. 2024-11-10T15:17:08,226 DEBUG [Thread-584 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:08,227 DEBUG [Thread-584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:08,227 DEBUG [Thread-584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:08,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:08,231 INFO [Thread-590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-590. 2024-11-10T15:17:08,231 DEBUG [Thread-590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:08,231 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:08,231 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:08,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:08,232 INFO [Thread-586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-586. 2024-11-10T15:17:08,232 DEBUG [Thread-586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:08,233 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:08,233 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:08,233 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:08,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/0722931b07464df8ad537ee001e7d9c9 is 50, key is test_row_0/C:col10/1731251827985/Put/seqid=0 2024-11-10T15:17:08,235 INFO [Thread-588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-588. 2024-11-10T15:17:08,235 DEBUG [Thread-588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:08,235 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:08,235 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:08,235 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:08,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742075_1251 (size=12301) 2024-11-10T15:17:08,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/0722931b07464df8ad537ee001e7d9c9 2024-11-10T15:17:08,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/dfe3a8bde81f4d218db4436bf9417800 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/dfe3a8bde81f4d218db4436bf9417800 2024-11-10T15:17:08,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/dfe3a8bde81f4d218db4436bf9417800, entries=250, sequenceid=459, filesize=47.5 K 2024-11-10T15:17:08,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/4c2475b3e7334bb784c7cbb487a0e122 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/4c2475b3e7334bb784c7cbb487a0e122 2024-11-10T15:17:08,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/4c2475b3e7334bb784c7cbb487a0e122, entries=150, sequenceid=459, filesize=12.0 K 2024-11-10T15:17:08,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/0722931b07464df8ad537ee001e7d9c9 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/0722931b07464df8ad537ee001e7d9c9 2024-11-10T15:17:08,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/0722931b07464df8ad537ee001e7d9c9, entries=150, sequenceid=459, filesize=12.0 K 2024-11-10T15:17:08,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7c1cb26053f9cb07691f27c558e31c12 in 305ms, sequenceid=459, compaction requested=false 2024-11-10T15:17:08,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:08,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T15:17:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:08,318 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T15:17:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:17:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:17:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:17:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:08,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111011578f019fc241c48040c7e4fe669ab7_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251828024/Put/seqid=0 2024-11-10T15:17:08,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742076_1252 (size=12454) 2024-11-10T15:17:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:08,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. as already flushing 2024-11-10T15:17:08,364 INFO [Thread-575 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-575. 2024-11-10T15:17:08,364 INFO [Thread-573 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-573. 2024-11-10T15:17:08,364 DEBUG [Thread-573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:08,364 DEBUG [Thread-575 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:08,364 DEBUG [Thread-573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:08,364 DEBUG [Thread-575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:08,364 DEBUG [Thread-573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:08,364 DEBUG [Thread-575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:08,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:08,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:08,366 INFO [Thread-577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-577. 2024-11-10T15:17:08,366 DEBUG [Thread-577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:08,366 DEBUG [Thread-577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:08,366 DEBUG [Thread-577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:08,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:08,368 INFO [Thread-581 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-581. 2024-11-10T15:17:08,368 DEBUG [Thread-581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:08,368 DEBUG [Thread-581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:08,369 DEBUG [Thread-581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:08,369 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:08,371 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/e211724178ad4390a79fdf55fd2fa20d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e211724178ad4390a79fdf55fd2fa20d 2024-11-10T15:17:08,378 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c1cb26053f9cb07691f27c558e31c12/B of 7c1cb26053f9cb07691f27c558e31c12 into e211724178ad4390a79fdf55fd2fa20d(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:08,378 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:08,378 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., storeName=7c1cb26053f9cb07691f27c558e31c12/B, priority=13, startTime=1731251827886; duration=0sec 2024-11-10T15:17:08,378 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:08,378 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c1cb26053f9cb07691f27c558e31c12:B 2024-11-10T15:17:08,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:08,737 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111011578f019fc241c48040c7e4fe669ab7_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111011578f019fc241c48040c7e4fe669ab7_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:08,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/694a8495a1d34a7cac19d4ae16ef106b, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:08,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/694a8495a1d34a7cac19d4ae16ef106b is 175, key is test_row_0/A:col10/1731251828024/Put/seqid=0 2024-11-10T15:17:08,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742077_1253 (size=31255) 2024-11-10T15:17:08,775 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=474, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/694a8495a1d34a7cac19d4ae16ef106b 2024-11-10T15:17:08,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/e74f4219be9d40499045a6de084058dc is 50, key is test_row_0/B:col10/1731251828024/Put/seqid=0 2024-11-10T15:17:08,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742078_1254 (size=12301) 2024-11-10T15:17:08,833 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/e74f4219be9d40499045a6de084058dc 2024-11-10T15:17:08,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/a09906b1639d46f7b80f85ffd1f12917 is 50, key is test_row_0/C:col10/1731251828024/Put/seqid=0 2024-11-10T15:17:08,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742079_1255 (size=12301) 2024-11-10T15:17:08,859 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/a09906b1639d46f7b80f85ffd1f12917 2024-11-10T15:17:08,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/694a8495a1d34a7cac19d4ae16ef106b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/694a8495a1d34a7cac19d4ae16ef106b 2024-11-10T15:17:08,881 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/694a8495a1d34a7cac19d4ae16ef106b, entries=150, sequenceid=474, filesize=30.5 K 2024-11-10T15:17:08,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/e74f4219be9d40499045a6de084058dc as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e74f4219be9d40499045a6de084058dc 2024-11-10T15:17:08,897 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e74f4219be9d40499045a6de084058dc, entries=150, sequenceid=474, filesize=12.0 K 2024-11-10T15:17:08,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/a09906b1639d46f7b80f85ffd1f12917 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a09906b1639d46f7b80f85ffd1f12917 2024-11-10T15:17:08,909 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a09906b1639d46f7b80f85ffd1f12917, entries=150, sequenceid=474, filesize=12.0 K 2024-11-10T15:17:08,910 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for 7c1cb26053f9cb07691f27c558e31c12 in 592ms, sequenceid=474, compaction requested=true 2024-11-10T15:17:08,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 7c1cb26053f9cb07691f27c558e31c12: 2024-11-10T15:17:08,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:08,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-10T15:17:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-10T15:17:08,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-10T15:17:08,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6960 sec 2024-11-10T15:17:08,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 3.7030 sec 2024-11-10T15:17:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T15:17:09,382 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:09,425 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T15:17:13,923 INFO [Thread-579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-579. 2024-11-10T15:17:13,923 DEBUG [Thread-579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:13,923 DEBUG [Thread-579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:13,923 DEBUG [Thread-579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:13,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:13,923 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T15:17:13,923 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-10T15:17:13,923 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-10T15:17:13,923 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-10T15:17:13,924 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-10T15:17:13,924 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-10T15:17:13,924 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T15:17:13,924 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2150 2024-11-10T15:17:13,924 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2064 2024-11-10T15:17:13,924 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T15:17:13,924 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 951 2024-11-10T15:17:13,924 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2852 rows 2024-11-10T15:17:13,924 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 961 2024-11-10T15:17:13,924 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2882 rows 2024-11-10T15:17:13,924 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T15:17:13,924 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T15:17:13,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:13,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:13,924 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:13,925 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T15:17:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T15:17:13,928 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251833927"}]},"ts":"1731251833927"} 2024-11-10T15:17:13,929 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T15:17:13,929 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T15:17:13,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T15:17:13,932 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c1cb26053f9cb07691f27c558e31c12, UNASSIGN}] 2024-11-10T15:17:13,932 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c1cb26053f9cb07691f27c558e31c12, UNASSIGN 2024-11-10T15:17:13,933 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=7c1cb26053f9cb07691f27c558e31c12, regionState=CLOSING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:17:13,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c1cb26053f9cb07691f27c558e31c12, UNASSIGN because future has completed 2024-11-10T15:17:13,934 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T15:17:13,934 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:17:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T15:17:14,086 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:14,087 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T15:17:14,087 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 7c1cb26053f9cb07691f27c558e31c12, disabling compactions & flushes 2024-11-10T15:17:14,087 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:14,087 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:14,087 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. after waiting 0 ms 2024-11-10T15:17:14,087 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:14,087 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 7c1cb26053f9cb07691f27c558e31c12 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T15:17:14,087 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=A 2024-11-10T15:17:14,087 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:14,087 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=B 2024-11-10T15:17:14,087 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:14,087 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c1cb26053f9cb07691f27c558e31c12, store=C 2024-11-10T15:17:14,087 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:14,093 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111046a770d5c0594c89a84f3b1e73f70141_7c1cb26053f9cb07691f27c558e31c12 is 50, key is test_row_0/A:col10/1731251833921/Put/seqid=0 2024-11-10T15:17:14,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742080_1256 (size=9914) 2024-11-10T15:17:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T15:17:14,499 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:14,506 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111046a770d5c0594c89a84f3b1e73f70141_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111046a770d5c0594c89a84f3b1e73f70141_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:14,507 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/1b8c8b06d4af4238a6ff41f89e50f628, store: [table=TestAcidGuarantees family=A region=7c1cb26053f9cb07691f27c558e31c12] 2024-11-10T15:17:14,507 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/1b8c8b06d4af4238a6ff41f89e50f628 is 175, key is test_row_0/A:col10/1731251833921/Put/seqid=0 2024-11-10T15:17:14,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742081_1257 (size=22561) 2024-11-10T15:17:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T15:17:14,914 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=483, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/1b8c8b06d4af4238a6ff41f89e50f628 2024-11-10T15:17:14,921 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/bd316b5b3c114e77b8b348d68dc9d8e9 is 50, key is test_row_0/B:col10/1731251833921/Put/seqid=0 2024-11-10T15:17:14,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742082_1258 (size=9857) 2024-11-10T15:17:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T15:17:15,329 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/bd316b5b3c114e77b8b348d68dc9d8e9 2024-11-10T15:17:15,336 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/a350aae2f34748debcda3e78aef95b27 is 50, key is test_row_0/C:col10/1731251833921/Put/seqid=0 2024-11-10T15:17:15,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742083_1259 (size=9857) 2024-11-10T15:17:15,340 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/a350aae2f34748debcda3e78aef95b27 2024-11-10T15:17:15,345 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/A/1b8c8b06d4af4238a6ff41f89e50f628 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1b8c8b06d4af4238a6ff41f89e50f628 2024-11-10T15:17:15,352 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1b8c8b06d4af4238a6ff41f89e50f628, entries=100, sequenceid=483, filesize=22.0 K 2024-11-10T15:17:15,353 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/B/bd316b5b3c114e77b8b348d68dc9d8e9 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/bd316b5b3c114e77b8b348d68dc9d8e9 2024-11-10T15:17:15,360 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/bd316b5b3c114e77b8b348d68dc9d8e9, entries=100, sequenceid=483, filesize=9.6 K 2024-11-10T15:17:15,361 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/.tmp/C/a350aae2f34748debcda3e78aef95b27 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a350aae2f34748debcda3e78aef95b27 2024-11-10T15:17:15,366 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a350aae2f34748debcda3e78aef95b27, entries=100, sequenceid=483, filesize=9.6 K 2024-11-10T15:17:15,368 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 7c1cb26053f9cb07691f27c558e31c12 in 1280ms, sequenceid=483, compaction requested=true 2024-11-10T15:17:15,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/8d3eba0879ea422ca53958c3a520acc4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/edfa00bd356043078b1d8fdb3e415837, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/4196dc98e4694744be8e34a53b037c1c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/e0f2e06a8c594a62a61f0ea8b8503f4f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/991bac5da9dc43c684686e3b1c9da19b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a6eb6565ee0840cda75f7ed4e8f035a0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/8fe7d16867df45f3b1329eb05fece8cc, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a8d0de89a7704165a203ca8bda4b0250, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/3fc20983e6d7429ba632c50955a98dca, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/628e995c68f34be5a8368170a121c88f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/0a5100b275464fd5a3be8e7bae39e4f1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/4c251a52d2e24dcea7055c54c9bfbd78, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/61e58ae9f9884835878646b727572ff8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/70353ecee1c347119090ca62180f8152, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/508491b4d04e4705a3e2e9cfcac12460, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/40c2207c398b46ceb1face1fb047ddb9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/b4b09fed1d1e439e975a1e9e8f05c304, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/fd3d19fafd8c467391d5670bc1ef19d9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2ab810755a174210bf64046886c9ba41, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2e7f71a1cedb4dbdb75cefafd762ff61, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/53553646695141adb0731c26cbcf7fc1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/cd9c44793b1c49d7bde737bdf416fe31, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1fb0c7c41dd748168dbbd13c68532aa3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/01ba6d70be5a4216b9ca88c5a092e9a7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1d784ebdd9b9480b88a0287d6be0aca5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/41a5f7614a6a46708938041af00ff295, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/6cbfc44744c54f6db6f359ae10d675f2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/7c0a2bd708e24368a74ececfcefa5065, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/e698554e212e4565a449fdd1e889282d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2236541118cb4022b7d637fec4ca8fc5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/269422976e93461fafce2fc4a19191f6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/83178f08f5df4b14b5ca62b0b714d3df, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a45749c8c845450fbc9784aff4199eaa] to archive 2024-11-10T15:17:15,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:17:15,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/8d3eba0879ea422ca53958c3a520acc4 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/8d3eba0879ea422ca53958c3a520acc4 2024-11-10T15:17:15,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/edfa00bd356043078b1d8fdb3e415837 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/edfa00bd356043078b1d8fdb3e415837 2024-11-10T15:17:15,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/4196dc98e4694744be8e34a53b037c1c to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/4196dc98e4694744be8e34a53b037c1c 2024-11-10T15:17:15,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/e0f2e06a8c594a62a61f0ea8b8503f4f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/e0f2e06a8c594a62a61f0ea8b8503f4f 2024-11-10T15:17:15,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/991bac5da9dc43c684686e3b1c9da19b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/991bac5da9dc43c684686e3b1c9da19b 2024-11-10T15:17:15,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a6eb6565ee0840cda75f7ed4e8f035a0 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a6eb6565ee0840cda75f7ed4e8f035a0 2024-11-10T15:17:15,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/8fe7d16867df45f3b1329eb05fece8cc to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/8fe7d16867df45f3b1329eb05fece8cc 2024-11-10T15:17:15,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a8d0de89a7704165a203ca8bda4b0250 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a8d0de89a7704165a203ca8bda4b0250 2024-11-10T15:17:15,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/3fc20983e6d7429ba632c50955a98dca to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/3fc20983e6d7429ba632c50955a98dca 2024-11-10T15:17:15,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/628e995c68f34be5a8368170a121c88f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/628e995c68f34be5a8368170a121c88f 2024-11-10T15:17:15,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/0a5100b275464fd5a3be8e7bae39e4f1 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/0a5100b275464fd5a3be8e7bae39e4f1 2024-11-10T15:17:15,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/4c251a52d2e24dcea7055c54c9bfbd78 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/4c251a52d2e24dcea7055c54c9bfbd78 2024-11-10T15:17:15,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/61e58ae9f9884835878646b727572ff8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/61e58ae9f9884835878646b727572ff8 2024-11-10T15:17:15,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/70353ecee1c347119090ca62180f8152 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/70353ecee1c347119090ca62180f8152 2024-11-10T15:17:15,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/508491b4d04e4705a3e2e9cfcac12460 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/508491b4d04e4705a3e2e9cfcac12460 2024-11-10T15:17:15,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/40c2207c398b46ceb1face1fb047ddb9 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/40c2207c398b46ceb1face1fb047ddb9 2024-11-10T15:17:15,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/b4b09fed1d1e439e975a1e9e8f05c304 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/b4b09fed1d1e439e975a1e9e8f05c304 2024-11-10T15:17:15,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/fd3d19fafd8c467391d5670bc1ef19d9 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/fd3d19fafd8c467391d5670bc1ef19d9 2024-11-10T15:17:15,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2ab810755a174210bf64046886c9ba41 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2ab810755a174210bf64046886c9ba41 2024-11-10T15:17:15,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2e7f71a1cedb4dbdb75cefafd762ff61 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2e7f71a1cedb4dbdb75cefafd762ff61 2024-11-10T15:17:15,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/53553646695141adb0731c26cbcf7fc1 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/53553646695141adb0731c26cbcf7fc1 2024-11-10T15:17:15,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/cd9c44793b1c49d7bde737bdf416fe31 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/cd9c44793b1c49d7bde737bdf416fe31 2024-11-10T15:17:15,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1fb0c7c41dd748168dbbd13c68532aa3 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1fb0c7c41dd748168dbbd13c68532aa3 2024-11-10T15:17:15,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/01ba6d70be5a4216b9ca88c5a092e9a7 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/01ba6d70be5a4216b9ca88c5a092e9a7 2024-11-10T15:17:15,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1d784ebdd9b9480b88a0287d6be0aca5 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1d784ebdd9b9480b88a0287d6be0aca5 2024-11-10T15:17:15,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/41a5f7614a6a46708938041af00ff295 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/41a5f7614a6a46708938041af00ff295 2024-11-10T15:17:15,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/6cbfc44744c54f6db6f359ae10d675f2 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/6cbfc44744c54f6db6f359ae10d675f2 2024-11-10T15:17:15,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/7c0a2bd708e24368a74ececfcefa5065 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/7c0a2bd708e24368a74ececfcefa5065 2024-11-10T15:17:15,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/e698554e212e4565a449fdd1e889282d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/e698554e212e4565a449fdd1e889282d 2024-11-10T15:17:15,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2236541118cb4022b7d637fec4ca8fc5 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/2236541118cb4022b7d637fec4ca8fc5 2024-11-10T15:17:15,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/269422976e93461fafce2fc4a19191f6 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/269422976e93461fafce2fc4a19191f6 2024-11-10T15:17:15,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/83178f08f5df4b14b5ca62b0b714d3df to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/83178f08f5df4b14b5ca62b0b714d3df 2024-11-10T15:17:15,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a45749c8c845450fbc9784aff4199eaa to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/a45749c8c845450fbc9784aff4199eaa 2024-11-10T15:17:15,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/618e9e669b8945b689fbdd18c48ae688, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e0f12cc24bc845518f2f23b1ca64a623, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/dab37d7370974669a06493e1306d2a59, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/584209bbb9a2470e8211a0461daf0e18, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/20c4fb74db3e4862a0f2f40f9d8fa23f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/f87f196a65bc408f88b71c1ae2088688, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/00cc213f51664a49ad8d9125721a9da3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e4bb914ea7ae49cf87a231eab9fbd2fe, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e053615d3c104d998bfeec9b145d9c86, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cee73babd8fb4d6190eb1d9e07ac87c0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/2d4a62edb8cb4aa3a82d5f8e6995e619, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cdfc6e5c6f2e49eeb92460a9b76d0405, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/efa7ebb54d674fdaa739caba0ced0d7a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/270b235ba4964176a2dc3088485d5faf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cc0761912f35488387cd878492fb785e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/dfa5af95010b421d899b3749770a9a47, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/db710ec591a245a5a81e8f94c42ce709, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cd815a5b31c141348fde00cbdf5d9eaf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/7bb187b3f1204e769f166ad4d41ce978, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/26d5814e8c0149a2a9461c2141486568, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/d4ceb14a50ac40b087f0914cb0e40a42, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/6ea459440feb46d2879a0e7efef3ece9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/30fd456bc30c45d6b5b7950641c74576, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/c921722cbe4e48fd8a98adc653ec69c0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/f83c790fe38741b38d268e6e7f7ab467, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/277015bfc2964087a07dc1fb8fe05975, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/112513cf36524d8bb6070e6cfea98319, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/8fdacaf049e24494aafa580ae2d25d20, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/bf75df38e65543eea0ab3140e8fa3025, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/22a8453239e543beb12ccb749c365e70, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/614d747e30c04802a486f1330511fc28, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/4d1293f7d638446ebe2719769accb97a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/b29dc6fa6a4540b3a9f4151f8746423e] to archive 2024-11-10T15:17:15,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:17:15,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/618e9e669b8945b689fbdd18c48ae688 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/618e9e669b8945b689fbdd18c48ae688 2024-11-10T15:17:15,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e0f12cc24bc845518f2f23b1ca64a623 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e0f12cc24bc845518f2f23b1ca64a623 2024-11-10T15:17:15,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/dab37d7370974669a06493e1306d2a59 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/dab37d7370974669a06493e1306d2a59 2024-11-10T15:17:15,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/584209bbb9a2470e8211a0461daf0e18 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/584209bbb9a2470e8211a0461daf0e18 2024-11-10T15:17:15,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/20c4fb74db3e4862a0f2f40f9d8fa23f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/20c4fb74db3e4862a0f2f40f9d8fa23f 2024-11-10T15:17:15,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/f87f196a65bc408f88b71c1ae2088688 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/f87f196a65bc408f88b71c1ae2088688 2024-11-10T15:17:15,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/00cc213f51664a49ad8d9125721a9da3 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/00cc213f51664a49ad8d9125721a9da3 2024-11-10T15:17:15,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e4bb914ea7ae49cf87a231eab9fbd2fe to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e4bb914ea7ae49cf87a231eab9fbd2fe 2024-11-10T15:17:15,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e053615d3c104d998bfeec9b145d9c86 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e053615d3c104d998bfeec9b145d9c86 2024-11-10T15:17:15,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cee73babd8fb4d6190eb1d9e07ac87c0 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cee73babd8fb4d6190eb1d9e07ac87c0 2024-11-10T15:17:15,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/2d4a62edb8cb4aa3a82d5f8e6995e619 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/2d4a62edb8cb4aa3a82d5f8e6995e619 2024-11-10T15:17:15,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cdfc6e5c6f2e49eeb92460a9b76d0405 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cdfc6e5c6f2e49eeb92460a9b76d0405 2024-11-10T15:17:15,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/efa7ebb54d674fdaa739caba0ced0d7a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/efa7ebb54d674fdaa739caba0ced0d7a 2024-11-10T15:17:15,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/270b235ba4964176a2dc3088485d5faf to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/270b235ba4964176a2dc3088485d5faf 2024-11-10T15:17:15,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cc0761912f35488387cd878492fb785e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cc0761912f35488387cd878492fb785e 2024-11-10T15:17:15,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/dfa5af95010b421d899b3749770a9a47 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/dfa5af95010b421d899b3749770a9a47 2024-11-10T15:17:15,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/db710ec591a245a5a81e8f94c42ce709 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/db710ec591a245a5a81e8f94c42ce709 2024-11-10T15:17:15,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cd815a5b31c141348fde00cbdf5d9eaf to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/cd815a5b31c141348fde00cbdf5d9eaf 2024-11-10T15:17:15,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/7bb187b3f1204e769f166ad4d41ce978 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/7bb187b3f1204e769f166ad4d41ce978 2024-11-10T15:17:15,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/26d5814e8c0149a2a9461c2141486568 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/26d5814e8c0149a2a9461c2141486568 2024-11-10T15:17:15,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/d4ceb14a50ac40b087f0914cb0e40a42 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/d4ceb14a50ac40b087f0914cb0e40a42 2024-11-10T15:17:15,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/6ea459440feb46d2879a0e7efef3ece9 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/6ea459440feb46d2879a0e7efef3ece9 2024-11-10T15:17:15,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/30fd456bc30c45d6b5b7950641c74576 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/30fd456bc30c45d6b5b7950641c74576 2024-11-10T15:17:15,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/c921722cbe4e48fd8a98adc653ec69c0 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/c921722cbe4e48fd8a98adc653ec69c0 2024-11-10T15:17:15,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/f83c790fe38741b38d268e6e7f7ab467 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/f83c790fe38741b38d268e6e7f7ab467 2024-11-10T15:17:15,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/277015bfc2964087a07dc1fb8fe05975 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/277015bfc2964087a07dc1fb8fe05975 2024-11-10T15:17:15,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/112513cf36524d8bb6070e6cfea98319 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/112513cf36524d8bb6070e6cfea98319 2024-11-10T15:17:15,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/8fdacaf049e24494aafa580ae2d25d20 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/8fdacaf049e24494aafa580ae2d25d20 2024-11-10T15:17:15,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/bf75df38e65543eea0ab3140e8fa3025 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/bf75df38e65543eea0ab3140e8fa3025 2024-11-10T15:17:15,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/22a8453239e543beb12ccb749c365e70 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/22a8453239e543beb12ccb749c365e70 2024-11-10T15:17:15,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/614d747e30c04802a486f1330511fc28 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/614d747e30c04802a486f1330511fc28 2024-11-10T15:17:15,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/4d1293f7d638446ebe2719769accb97a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/4d1293f7d638446ebe2719769accb97a 2024-11-10T15:17:15,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/b29dc6fa6a4540b3a9f4151f8746423e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/b29dc6fa6a4540b3a9f4151f8746423e 2024-11-10T15:17:15,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/6b68ae09e5c64517abd50edaeac6a598, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d3ebe27c94674e05a05ba4ae1757b0a2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2a7ca3b0ff1a4e2ca16d2128352bd2f6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/23ac4329a03e44b98c48bd8bd4a28ed4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c238a2217c2f423c96fd8ccc1df39c3d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/9f467555ed2a4c6fb5898059a457082f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/e3c9c91e52f44cb1b07c0adc006b9def, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/b4ceffe8a006492e8c4b3d2c307f7e8e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/344f26407b534b309ffa9153713ff408, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/1b792d90406e46a49cfc24b964493b2f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d4a2ad61ccaf4e9a846bb2b4bfcdb848, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2ed27ce5852343e1ad9fe81c3b4ffdb9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c0fa1d359e3749cf9a2b61e25a9ddc42, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/ee29dcb294c1417daefe133e1132a8ba, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/36f901fde6e846e3af65fec7b1fdf02a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/1ca4b9559e6548e5bc3fc6166657b21b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/60081e71e4d445f88f1078ebbceac9d8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/e9673afec04f4c8d861131298b9b4835, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d9165abc3048496c9e52fc6f77cb3b96, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/94df78e65e554d3c8b921a61f75cb5d7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/8543997a132345ae9d8b0988d0ef9261, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/4bca5d496d4e4a399a73bea7a2fc9b95, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d2853ef679af4b40971b6e432c7d8511, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a319f8c3e7c84b5a993e7a3a14a0c5fc, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/0ef13013fc334e949b7263a0d29a926b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d189ba5b37b642dbb85b1b84e7b1836d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/556204548f2a497e8ff9ddc66dfee94e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/69843d87e59c49478f38af829d5c55b8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2093820aa2a84420a118359a7041d140, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d3ae7201012044e189e8589b746fb159, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a2b81ad4844748a5a301b1864984b838, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c53864e8e9ac4fbdb0f0c1f3234dbd92, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/4d92aae6dded4fa9a32ca5ae9727ff8a] to archive 2024-11-10T15:17:15,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:17:15,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/6b68ae09e5c64517abd50edaeac6a598 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/6b68ae09e5c64517abd50edaeac6a598 2024-11-10T15:17:15,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d3ebe27c94674e05a05ba4ae1757b0a2 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d3ebe27c94674e05a05ba4ae1757b0a2 2024-11-10T15:17:15,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2a7ca3b0ff1a4e2ca16d2128352bd2f6 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2a7ca3b0ff1a4e2ca16d2128352bd2f6 2024-11-10T15:17:15,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/23ac4329a03e44b98c48bd8bd4a28ed4 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/23ac4329a03e44b98c48bd8bd4a28ed4 2024-11-10T15:17:15,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c238a2217c2f423c96fd8ccc1df39c3d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c238a2217c2f423c96fd8ccc1df39c3d 2024-11-10T15:17:15,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/9f467555ed2a4c6fb5898059a457082f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/9f467555ed2a4c6fb5898059a457082f 2024-11-10T15:17:15,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/e3c9c91e52f44cb1b07c0adc006b9def to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/e3c9c91e52f44cb1b07c0adc006b9def 2024-11-10T15:17:15,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/b4ceffe8a006492e8c4b3d2c307f7e8e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/b4ceffe8a006492e8c4b3d2c307f7e8e 2024-11-10T15:17:15,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/344f26407b534b309ffa9153713ff408 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/344f26407b534b309ffa9153713ff408 2024-11-10T15:17:15,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/1b792d90406e46a49cfc24b964493b2f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/1b792d90406e46a49cfc24b964493b2f 2024-11-10T15:17:15,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d4a2ad61ccaf4e9a846bb2b4bfcdb848 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d4a2ad61ccaf4e9a846bb2b4bfcdb848 2024-11-10T15:17:15,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2ed27ce5852343e1ad9fe81c3b4ffdb9 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2ed27ce5852343e1ad9fe81c3b4ffdb9 2024-11-10T15:17:15,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c0fa1d359e3749cf9a2b61e25a9ddc42 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c0fa1d359e3749cf9a2b61e25a9ddc42 2024-11-10T15:17:15,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/ee29dcb294c1417daefe133e1132a8ba to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/ee29dcb294c1417daefe133e1132a8ba 2024-11-10T15:17:15,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/36f901fde6e846e3af65fec7b1fdf02a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/36f901fde6e846e3af65fec7b1fdf02a 2024-11-10T15:17:15,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/1ca4b9559e6548e5bc3fc6166657b21b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/1ca4b9559e6548e5bc3fc6166657b21b 2024-11-10T15:17:15,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/60081e71e4d445f88f1078ebbceac9d8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/60081e71e4d445f88f1078ebbceac9d8 2024-11-10T15:17:15,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/e9673afec04f4c8d861131298b9b4835 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/e9673afec04f4c8d861131298b9b4835 2024-11-10T15:17:15,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d9165abc3048496c9e52fc6f77cb3b96 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d9165abc3048496c9e52fc6f77cb3b96 2024-11-10T15:17:15,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/94df78e65e554d3c8b921a61f75cb5d7 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/94df78e65e554d3c8b921a61f75cb5d7 2024-11-10T15:17:15,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/8543997a132345ae9d8b0988d0ef9261 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/8543997a132345ae9d8b0988d0ef9261 2024-11-10T15:17:15,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/4bca5d496d4e4a399a73bea7a2fc9b95 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/4bca5d496d4e4a399a73bea7a2fc9b95 2024-11-10T15:17:15,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d2853ef679af4b40971b6e432c7d8511 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d2853ef679af4b40971b6e432c7d8511 2024-11-10T15:17:15,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a319f8c3e7c84b5a993e7a3a14a0c5fc to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a319f8c3e7c84b5a993e7a3a14a0c5fc 2024-11-10T15:17:15,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/0ef13013fc334e949b7263a0d29a926b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/0ef13013fc334e949b7263a0d29a926b 2024-11-10T15:17:15,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d189ba5b37b642dbb85b1b84e7b1836d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d189ba5b37b642dbb85b1b84e7b1836d 2024-11-10T15:17:15,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/556204548f2a497e8ff9ddc66dfee94e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/556204548f2a497e8ff9ddc66dfee94e 2024-11-10T15:17:15,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/69843d87e59c49478f38af829d5c55b8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/69843d87e59c49478f38af829d5c55b8 2024-11-10T15:17:15,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2093820aa2a84420a118359a7041d140 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/2093820aa2a84420a118359a7041d140 2024-11-10T15:17:15,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d3ae7201012044e189e8589b746fb159 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/d3ae7201012044e189e8589b746fb159 2024-11-10T15:17:15,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a2b81ad4844748a5a301b1864984b838 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a2b81ad4844748a5a301b1864984b838 2024-11-10T15:17:15,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c53864e8e9ac4fbdb0f0c1f3234dbd92 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/c53864e8e9ac4fbdb0f0c1f3234dbd92 2024-11-10T15:17:15,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/4d92aae6dded4fa9a32ca5ae9727ff8a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/4d92aae6dded4fa9a32ca5ae9727ff8a 2024-11-10T15:17:15,581 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/recovered.edits/486.seqid, newMaxSeqId=486, maxSeqId=4 2024-11-10T15:17:15,582 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. 2024-11-10T15:17:15,582 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 7c1cb26053f9cb07691f27c558e31c12: Waiting for close lock at 1731251834087Running coprocessor pre-close hooks at 1731251834087Disabling compacts and flushes for region at 1731251834087Disabling writes for close at 1731251834087Obtaining lock to block concurrent updates at 1731251834087Preparing flush snapshotting stores in 7c1cb26053f9cb07691f27c558e31c12 at 1731251834087Finished memstore snapshotting TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731251834087Flushing stores of TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12. at 1731251834088 (+1 ms)Flushing 7c1cb26053f9cb07691f27c558e31c12/A: creating writer at 1731251834088Flushing mob file 7c1cb26053f9cb07691f27c558e31c12/A: appending metadata at 1731251834093 (+5 ms)Flushing mob file 7c1cb26053f9cb07691f27c558e31c12/A: closing flushed file at 1731251834093Flushing 7c1cb26053f9cb07691f27c558e31c12/A: appending metadata at 1731251834507 (+414 ms)Flushing 7c1cb26053f9cb07691f27c558e31c12/A: closing flushed file at 1731251834507Flushing 7c1cb26053f9cb07691f27c558e31c12/B: creating writer at 1731251834917 (+410 ms)Flushing 7c1cb26053f9cb07691f27c558e31c12/B: appending metadata at 1731251834920 (+3 ms)Flushing 7c1cb26053f9cb07691f27c558e31c12/B: closing flushed file at 1731251834920Flushing 7c1cb26053f9cb07691f27c558e31c12/C: creating writer at 1731251835334 (+414 ms)Flushing 7c1cb26053f9cb07691f27c558e31c12/C: appending metadata at 1731251835336 (+2 ms)Flushing 7c1cb26053f9cb07691f27c558e31c12/C: closing flushed file at 1731251835336Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58a002de: reopening flushed file at 1731251835344 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fca6003: reopening flushed file at 1731251835352 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5de7ea45: reopening flushed file at 1731251835360 (+8 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 7c1cb26053f9cb07691f27c558e31c12 in 1280ms, sequenceid=483, compaction requested=true at 1731251835368 (+8 ms)Writing region close event to WAL at 1731251835577 (+209 ms)Running coprocessor post-close hooks at 1731251835582 (+5 ms)Closed at 1731251835582 2024-11-10T15:17:15,584 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:15,585 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=7c1cb26053f9cb07691f27c558e31c12, regionState=CLOSED 2024-11-10T15:17:15,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:17:15,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-10T15:17:15,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 7c1cb26053f9cb07691f27c558e31c12, server=c0771061be61,46087,1731251771284 in 1.6540 sec 2024-11-10T15:17:15,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-10T15:17:15,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c1cb26053f9cb07691f27c558e31c12, UNASSIGN in 1.6570 sec 2024-11-10T15:17:15,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-10T15:17:15,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6620 sec 2024-11-10T15:17:15,594 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251835594"}]},"ts":"1731251835594"} 2024-11-10T15:17:15,596 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T15:17:15,596 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T15:17:15,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6710 sec 2024-11-10T15:17:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T15:17:16,072 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:16,072 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T15:17:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:16,074 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T15:17:16,075 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:16,077 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,080 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/recovered.edits] 2024-11-10T15:17:16,082 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1b8c8b06d4af4238a6ff41f89e50f628 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/1b8c8b06d4af4238a6ff41f89e50f628 2024-11-10T15:17:16,083 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/24a879e5e74c44ab8365fb9753fdecec to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/24a879e5e74c44ab8365fb9753fdecec 2024-11-10T15:17:16,084 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/694a8495a1d34a7cac19d4ae16ef106b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/694a8495a1d34a7cac19d4ae16ef106b 2024-11-10T15:17:16,085 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/dfe3a8bde81f4d218db4436bf9417800 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/A/dfe3a8bde81f4d218db4436bf9417800 2024-11-10T15:17:16,087 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/4c2475b3e7334bb784c7cbb487a0e122 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/4c2475b3e7334bb784c7cbb487a0e122 2024-11-10T15:17:16,089 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/bd316b5b3c114e77b8b348d68dc9d8e9 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/bd316b5b3c114e77b8b348d68dc9d8e9 2024-11-10T15:17:16,090 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e211724178ad4390a79fdf55fd2fa20d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e211724178ad4390a79fdf55fd2fa20d 2024-11-10T15:17:16,091 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e74f4219be9d40499045a6de084058dc to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/B/e74f4219be9d40499045a6de084058dc 2024-11-10T15:17:16,094 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/0722931b07464df8ad537ee001e7d9c9 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/0722931b07464df8ad537ee001e7d9c9 2024-11-10T15:17:16,095 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a09906b1639d46f7b80f85ffd1f12917 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a09906b1639d46f7b80f85ffd1f12917 2024-11-10T15:17:16,096 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a350aae2f34748debcda3e78aef95b27 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/a350aae2f34748debcda3e78aef95b27 2024-11-10T15:17:16,097 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/aeaecccc8f584c1fae8e1aee98ec0916 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/C/aeaecccc8f584c1fae8e1aee98ec0916 2024-11-10T15:17:16,099 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/recovered.edits/486.seqid to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12/recovered.edits/486.seqid 2024-11-10T15:17:16,100 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,100 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T15:17:16,100 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T15:17:16,101 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T15:17:16,104 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111011578f019fc241c48040c7e4fe669ab7_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111011578f019fc241c48040c7e4fe669ab7_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,106 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111011b310eccf7b42a98533ba0303c7ad99_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111011b310eccf7b42a98533ba0303c7ad99_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,107 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111016ebf519ac4d406cb8647b094fe39595_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111016ebf519ac4d406cb8647b094fe39595_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,109 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111019019805e14346b1a8a3a3e5d162bc51_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111019019805e14346b1a8a3a3e5d162bc51_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,110 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111019aa6e77513e460daf2514e7f0a691f3_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111019aa6e77513e460daf2514e7f0a691f3_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,112 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111021a5294f9138443d943915edbd7a1ef1_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111021a5294f9138443d943915edbd7a1ef1_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,113 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102b8f993073b84e50ab71062646f6c108_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102b8f993073b84e50ab71062646f6c108_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,115 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111030222bfd494349cd9e850d7e3bd61e11_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111030222bfd494349cd9e850d7e3bd61e11_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,116 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111043e2c3997da2494eaeaba3a5fefddca6_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111043e2c3997da2494eaeaba3a5fefddca6_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,118 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111046a770d5c0594c89a84f3b1e73f70141_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111046a770d5c0594c89a84f3b1e73f70141_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,119 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111051068fd7900f484ab9cc43b576631154_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111051068fd7900f484ab9cc43b576631154_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,121 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110512d4f445d5b4bdeb4c88d27c7f0f4d9_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110512d4f445d5b4bdeb4c88d27c7f0f4d9_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,122 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105498c502ce2e4609a409e520b968880f_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105498c502ce2e4609a409e520b968880f_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,123 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105651d694fa324ad2bc24d25edd9fede3_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105651d694fa324ad2bc24d25edd9fede3_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,124 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111061294b6e496b4912bbfdf6d0d75e9c90_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111061294b6e496b4912bbfdf6d0d75e9c90_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,125 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107509ba075e3d496596344575fddf3bc3_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107509ba075e3d496596344575fddf3bc3_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,126 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110781f3364a8f248c397337534763531c3_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110781f3364a8f248c397337534763531c3_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,127 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107b85a342ec634515adfc9b873ff0d671_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107b85a342ec634515adfc9b873ff0d671_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,128 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107d0b6c74010e43b3ae93c343f87b0224_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107d0b6c74010e43b3ae93c343f87b0224_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,129 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a40046b8cb7540ac919162cca91965e6_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a40046b8cb7540ac919162cca91965e6_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,130 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110aa68cf23dc434794b0da182414f6b791_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110aa68cf23dc434794b0da182414f6b791_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,131 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b5c1cd1e1e3c452c879334710d5214fb_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b5c1cd1e1e3c452c879334710d5214fb_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,132 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d85c0ef66f9440409dc1c625e1ad5278_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d85c0ef66f9440409dc1c625e1ad5278_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,133 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d8f4accc65b040dfae9584aea9584b71_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d8f4accc65b040dfae9584aea9584b71_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,134 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e945b0e740cf42299d52dd21738d3b96_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e945b0e740cf42299d52dd21738d3b96_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,136 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f29136ae47ba4e11bf66c71c332ec592_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f29136ae47ba4e11bf66c71c332ec592_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,137 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ff4d22e3a6ba40cc8fc6fe8d9774a684_7c1cb26053f9cb07691f27c558e31c12 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ff4d22e3a6ba40cc8fc6fe8d9774a684_7c1cb26053f9cb07691f27c558e31c12 2024-11-10T15:17:16,138 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T15:17:16,141 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:16,144 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T15:17:16,147 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T15:17:16,149 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:16,150 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T15:17:16,151 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731251836150"}]},"ts":"9223372036854775807"} 2024-11-10T15:17:16,160 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T15:17:16,160 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7c1cb26053f9cb07691f27c558e31c12, NAME => 'TestAcidGuarantees,,1731251805041.7c1cb26053f9cb07691f27c558e31c12.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T15:17:16,160 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T15:17:16,161 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731251836160"}]},"ts":"9223372036854775807"} 2024-11-10T15:17:16,168 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T15:17:16,173 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:16,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 101 msec 2024-11-10T15:17:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T15:17:16,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T15:17:16,182 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:16,212 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 246) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1405295892_22 at /127.0.0.1:42942 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/cluster_33e785d3-c89b-58a2-b49e-d3558fd8bba1/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/cluster_33e785d3-c89b-58a2-b49e-d3558fd8bba1/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=808 (was 773) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2850 (was 3691) 2024-11-10T15:17:16,233 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=808, ProcessCount=11, AvailableMemoryMB=2850 2024-11-10T15:17:16,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T15:17:16,235 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T15:17:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:16,243 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T15:17:16,243 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:16,245 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-10T15:17:16,250 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T15:17:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T15:17:16,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742084_1260 (size=1061) 2024-11-10T15:17:16,298 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85 2024-11-10T15:17:16,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742085_1261 (size=53) 2024-11-10T15:17:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T15:17:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T15:17:16,728 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:17:16,728 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c42fcce8c0425e5889d52562aa8da5f6, disabling compactions & flushes 2024-11-10T15:17:16,728 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:16,728 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:16,728 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. after waiting 0 ms 2024-11-10T15:17:16,728 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:16,728 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:16,729 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c42fcce8c0425e5889d52562aa8da5f6: Waiting for close lock at 1731251836728Disabling compacts and flushes for region at 1731251836728Disabling writes for close at 1731251836728Writing region close event to WAL at 1731251836728Closed at 1731251836728 2024-11-10T15:17:16,731 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T15:17:16,732 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731251836731"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731251836731"}]},"ts":"1731251836731"} 2024-11-10T15:17:16,733 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T15:17:16,734 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T15:17:16,735 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251836734"}]},"ts":"1731251836734"} 2024-11-10T15:17:16,736 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T15:17:16,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c42fcce8c0425e5889d52562aa8da5f6, ASSIGN}] 2024-11-10T15:17:16,738 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c42fcce8c0425e5889d52562aa8da5f6, ASSIGN 2024-11-10T15:17:16,739 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c42fcce8c0425e5889d52562aa8da5f6, ASSIGN; state=OFFLINE, location=c0771061be61,46087,1731251771284; forceNewPlan=false, retain=false 2024-11-10T15:17:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T15:17:16,889 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=c42fcce8c0425e5889d52562aa8da5f6, regionState=OPENING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:17:16,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c42fcce8c0425e5889d52562aa8da5f6, ASSIGN because future has completed 2024-11-10T15:17:16,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:17:17,047 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:17,047 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} 2024-11-10T15:17:17,048 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:17,048 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:17:17,048 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:17,048 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:17,050 INFO [StoreOpener-c42fcce8c0425e5889d52562aa8da5f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:17,051 INFO [StoreOpener-c42fcce8c0425e5889d52562aa8da5f6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:17:17,052 INFO [StoreOpener-c42fcce8c0425e5889d52562aa8da5f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c42fcce8c0425e5889d52562aa8da5f6 columnFamilyName A 2024-11-10T15:17:17,052 DEBUG [StoreOpener-c42fcce8c0425e5889d52562aa8da5f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:17,052 INFO [StoreOpener-c42fcce8c0425e5889d52562aa8da5f6-1 {}] regionserver.HStore(327): Store=c42fcce8c0425e5889d52562aa8da5f6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:17:17,053 INFO [StoreOpener-c42fcce8c0425e5889d52562aa8da5f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:17,054 INFO [StoreOpener-c42fcce8c0425e5889d52562aa8da5f6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:17:17,055 INFO [StoreOpener-c42fcce8c0425e5889d52562aa8da5f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c42fcce8c0425e5889d52562aa8da5f6 columnFamilyName B 2024-11-10T15:17:17,055 DEBUG [StoreOpener-c42fcce8c0425e5889d52562aa8da5f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:17,055 INFO [StoreOpener-c42fcce8c0425e5889d52562aa8da5f6-1 {}] regionserver.HStore(327): Store=c42fcce8c0425e5889d52562aa8da5f6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:17:17,056 INFO [StoreOpener-c42fcce8c0425e5889d52562aa8da5f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:17,058 INFO [StoreOpener-c42fcce8c0425e5889d52562aa8da5f6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:17:17,059 INFO [StoreOpener-c42fcce8c0425e5889d52562aa8da5f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c42fcce8c0425e5889d52562aa8da5f6 columnFamilyName C 2024-11-10T15:17:17,059 DEBUG [StoreOpener-c42fcce8c0425e5889d52562aa8da5f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:17,059 INFO [StoreOpener-c42fcce8c0425e5889d52562aa8da5f6-1 {}] regionserver.HStore(327): Store=c42fcce8c0425e5889d52562aa8da5f6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:17:17,059 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:17,060 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:17,060 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:17,061 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:17,062 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:17,062 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:17,063 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T15:17:17,064 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:17,067 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T15:17:17,067 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened c42fcce8c0425e5889d52562aa8da5f6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75495258, jitterRate=0.12496700882911682}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T15:17:17,067 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:17,068 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for c42fcce8c0425e5889d52562aa8da5f6: Running coprocessor pre-open hook at 1731251837048Writing region info on filesystem at 1731251837048Initializing all the Stores at 1731251837049 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251837049Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251837049Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251837050 (+1 ms)Cleaning up temporary data from old regions at 1731251837062 (+12 ms)Running coprocessor post-open hooks at 1731251837067 (+5 ms)Region opened successfully at 1731251837068 (+1 ms) 2024-11-10T15:17:17,069 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., pid=65, masterSystemTime=1731251837044 2024-11-10T15:17:17,071 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:17,071 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:17,071 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=c42fcce8c0425e5889d52562aa8da5f6, regionState=OPEN, openSeqNum=2, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:17:17,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:17:17,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-10T15:17:17,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 in 182 msec 2024-11-10T15:17:17,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-10T15:17:17,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c42fcce8c0425e5889d52562aa8da5f6, ASSIGN in 338 msec 2024-11-10T15:17:17,077 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T15:17:17,077 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251837077"}]},"ts":"1731251837077"} 2024-11-10T15:17:17,078 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T15:17:17,079 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T15:17:17,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 844 msec 2024-11-10T15:17:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T15:17:17,391 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:17,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,392 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:17,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:17,394 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44862f47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:17,396 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4905445e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,397 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:17,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:17,399 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T15:17:17,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is c0771061be61,41501,1731251770419 2024-11-10T15:17:17,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@37284d9b 2024-11-10T15:17:17,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T15:17:17,401 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T15:17:17,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:17,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:17,403 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:17,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:17,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658db990, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:17,405 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:17,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57b207f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,407 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:17,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:17,407 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce7111b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:17,409 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@233292ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,410 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:17,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:17,410 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ef19fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:17,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:17,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:17,411 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:17,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6688f697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,412 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:17,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:17,412 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e1fc172, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:17,413 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a797f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:17,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:17,415 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78863f3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:17,416 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18906f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,418 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:17,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:17,418 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67412388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:17,420 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e500ae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:17,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:17,421 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76cd9763, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:17,423 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:17,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f38d64a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:17,424 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:17,425 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e5c3c57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:17,426 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63455081, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,427 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:17,427 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:17,427 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b92c4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:17,429 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:17,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62a4045d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,430 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:17,430 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:17,430 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@459b9868, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:17,432 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@257d05c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:17,432 DEBUG [Thread-1221 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:17,432 DEBUG [Thread-1204 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:17,432 DEBUG [Thread-1210 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:17,433 DEBUG [Thread-1206 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:17,433 DEBUG [Thread-1219 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:17,433 DEBUG [Thread-1217 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:17,434 DEBUG [Thread-1202 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:17,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:17:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:17,435 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:17,435 DEBUG [Thread-1208 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:17,436 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:17,436 DEBUG [Thread-1215 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:17,436 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:17,436 DEBUG [Thread-1213 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:17,436 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:17,436 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:17:17,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:17,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:17:17,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:17,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:17:17,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:17:17,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-10T15:17:17,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T15:17:17,439 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:17,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:17:17,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:17:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:17:17,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:17,440 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:17:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:17:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:17:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:17:17,443 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:17:17,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:17:17,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:17,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T15:17:17,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:17,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:17,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:17,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:17,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:17,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:17,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/1d1285dde25e4e27aa685fae993a4ff5 is 50, key is test_row_0/A:col10/1731251837443/Put/seqid=0 2024-11-10T15:17:17,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251837466, max timestamp=1731251837477Num uniques 859; 2024-11-10T15:17:17,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251837466, max timestamp=1731251837477Num uniques 859; 2024-11-10T15:17:17,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:17:17,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:17:17,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251837466, max timestamp=1731251837477Num uniques 859; 2024-11-10T15:17:17,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:17:17,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742086_1262 (size=9657) 2024-11-10T15:17:17,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/1d1285dde25e4e27aa685fae993a4ff5 2024-11-10T15:17:17,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251897512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251897512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:17,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38552 deadline: 1731251897514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:17,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251897514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:17,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251897514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:17,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/1ec579dc1f4642528f2bb85c53afa5ce is 50, key is test_row_0/B:col10/1731251837443/Put/seqid=0 2024-11-10T15:17:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T15:17:17,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742087_1263 (size=9657) 2024-11-10T15:17:17,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/1ec579dc1f4642528f2bb85c53afa5ce 2024-11-10T15:17:17,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/14fb2f6a71a841f9ae109e4c026c6c66 is 50, key is test_row_0/C:col10/1731251837443/Put/seqid=0 2024-11-10T15:17:17,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T15:17:17,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:17,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:17,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:17,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:17,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:17,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742088_1264 (size=9657) 2024-11-10T15:17:17,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:17,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251897631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251897631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:17,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38552 deadline: 1731251897631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251897631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:17,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251897631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:17,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T15:17:17,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:17,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:17,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:17,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:17,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T15:17:17,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251897841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38552 deadline: 1731251897841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:17,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251897841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:17,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251897841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:17,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:17,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251897844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:17,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:17,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T15:17:17,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:17,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:17,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:17,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:17,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:18,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/14fb2f6a71a841f9ae109e4c026c6c66 2024-11-10T15:17:18,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/1d1285dde25e4e27aa685fae993a4ff5 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/1d1285dde25e4e27aa685fae993a4ff5 2024-11-10T15:17:18,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/1d1285dde25e4e27aa685fae993a4ff5, entries=100, sequenceid=12, filesize=9.4 K 2024-11-10T15:17:18,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/1ec579dc1f4642528f2bb85c53afa5ce as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/1ec579dc1f4642528f2bb85c53afa5ce 2024-11-10T15:17:18,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/1ec579dc1f4642528f2bb85c53afa5ce, entries=100, sequenceid=12, filesize=9.4 K 2024-11-10T15:17:18,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/14fb2f6a71a841f9ae109e4c026c6c66 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/14fb2f6a71a841f9ae109e4c026c6c66 2024-11-10T15:17:18,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/14fb2f6a71a841f9ae109e4c026c6c66, entries=100, sequenceid=12, filesize=9.4 K 2024-11-10T15:17:18,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c42fcce8c0425e5889d52562aa8da5f6 in 573ms, sequenceid=12, compaction requested=false 2024-11-10T15:17:18,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:18,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T15:17:18,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:18,056 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-10T15:17:18,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:18,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:18,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:18,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:18,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:18,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:18,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/e42cf7a9e15b431d850cf1b95a4964ad is 50, key is test_row_0/A:col10/1731251837511/Put/seqid=0 2024-11-10T15:17:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T15:17:18,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742089_1265 (size=12001) 2024-11-10T15:17:18,066 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/e42cf7a9e15b431d850cf1b95a4964ad 2024-11-10T15:17:18,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/b0d725ce8a32454fa5fa895633e45948 is 50, key is test_row_0/B:col10/1731251837511/Put/seqid=0 2024-11-10T15:17:18,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742090_1266 (size=12001) 2024-11-10T15:17:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:18,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:18,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251898158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251898159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251898159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251898160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38552 deadline: 1731251898161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251898271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251898271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251898272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251898272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,480 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/b0d725ce8a32454fa5fa895633e45948 2024-11-10T15:17:18,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251898482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251898482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251898483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251898483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/fce1c3814c14411f8954717fa70a6ba8 is 50, key is test_row_0/C:col10/1731251837511/Put/seqid=0 2024-11-10T15:17:18,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742091_1267 (size=12001) 2024-11-10T15:17:18,509 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/fce1c3814c14411f8954717fa70a6ba8 2024-11-10T15:17:18,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/e42cf7a9e15b431d850cf1b95a4964ad as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e42cf7a9e15b431d850cf1b95a4964ad 2024-11-10T15:17:18,521 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e42cf7a9e15b431d850cf1b95a4964ad, entries=150, sequenceid=47, filesize=11.7 K 2024-11-10T15:17:18,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/b0d725ce8a32454fa5fa895633e45948 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/b0d725ce8a32454fa5fa895633e45948 2024-11-10T15:17:18,527 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/b0d725ce8a32454fa5fa895633e45948, entries=150, sequenceid=47, filesize=11.7 K 2024-11-10T15:17:18,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/fce1c3814c14411f8954717fa70a6ba8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/fce1c3814c14411f8954717fa70a6ba8 2024-11-10T15:17:18,533 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/fce1c3814c14411f8954717fa70a6ba8, entries=150, sequenceid=47, filesize=11.7 K 2024-11-10T15:17:18,534 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for c42fcce8c0425e5889d52562aa8da5f6 in 479ms, sequenceid=47, compaction requested=false 2024-11-10T15:17:18,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:18,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:18,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-10T15:17:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-10T15:17:18,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-10T15:17:18,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0920 sec 2024-11-10T15:17:18,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.1030 sec 2024-11-10T15:17:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T15:17:18,572 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:18,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:17:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-10T15:17:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T15:17:18,575 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:17:18,576 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:17:18,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:17:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:18,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T15:17:18,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:18,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:18,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:18,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:18,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:18,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:18,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/d556a01c60e945708404461b4119c313 is 50, key is test_row_0/A:col10/1731251838158/Put/seqid=0 2024-11-10T15:17:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T15:17:18,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742092_1268 (size=14341) 2024-11-10T15:17:18,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T15:17:18,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:18,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:18,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:18,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:18,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:18,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38552 deadline: 1731251898749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251898791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251898791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251898791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251898791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38552 deadline: 1731251898862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:18,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:18,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:18,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T15:17:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:18,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T15:17:19,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T15:17:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38552 deadline: 1731251899072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/d556a01c60e945708404461b4119c313 2024-11-10T15:17:19,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/14037ffb6fb042e7a1871e3414200be0 is 50, key is test_row_0/B:col10/1731251838158/Put/seqid=0 2024-11-10T15:17:19,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742093_1269 (size=12001) 2024-11-10T15:17:19,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/14037ffb6fb042e7a1871e3414200be0 2024-11-10T15:17:19,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/b9f360ea5e6a400ba0d76f9df90cc73a is 50, key is test_row_0/C:col10/1731251838158/Put/seqid=0 2024-11-10T15:17:19,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742094_1270 (size=12001) 2024-11-10T15:17:19,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/b9f360ea5e6a400ba0d76f9df90cc73a 2024-11-10T15:17:19,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/d556a01c60e945708404461b4119c313 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/d556a01c60e945708404461b4119c313 2024-11-10T15:17:19,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T15:17:19,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:19,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/d556a01c60e945708404461b4119c313, entries=200, sequenceid=59, filesize=14.0 K 2024-11-10T15:17:19,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/14037ffb6fb042e7a1871e3414200be0 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/14037ffb6fb042e7a1871e3414200be0 2024-11-10T15:17:19,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/14037ffb6fb042e7a1871e3414200be0, entries=150, sequenceid=59, filesize=11.7 K 2024-11-10T15:17:19,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/b9f360ea5e6a400ba0d76f9df90cc73a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/b9f360ea5e6a400ba0d76f9df90cc73a 2024-11-10T15:17:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T15:17:19,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/b9f360ea5e6a400ba0d76f9df90cc73a, entries=150, sequenceid=59, filesize=11.7 K 2024-11-10T15:17:19,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c42fcce8c0425e5889d52562aa8da5f6 in 537ms, sequenceid=59, compaction requested=true 2024-11-10T15:17:19,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:19,210 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:17:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:17:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:19,211 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:17:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:19,212 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:19,212 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/A is initiating minor compaction (all files) 2024-11-10T15:17:19,212 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/A in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,212 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/1d1285dde25e4e27aa685fae993a4ff5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e42cf7a9e15b431d850cf1b95a4964ad, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/d556a01c60e945708404461b4119c313] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=35.2 K 2024-11-10T15:17:19,212 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:19,213 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/B is initiating minor compaction (all files) 2024-11-10T15:17:19,213 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/B in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,213 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/1ec579dc1f4642528f2bb85c53afa5ce, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/b0d725ce8a32454fa5fa895633e45948, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/14037ffb6fb042e7a1871e3414200be0] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=32.9 K 2024-11-10T15:17:19,214 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d1285dde25e4e27aa685fae993a4ff5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731251837443 2024-11-10T15:17:19,214 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ec579dc1f4642528f2bb85c53afa5ce, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731251837443 2024-11-10T15:17:19,214 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e42cf7a9e15b431d850cf1b95a4964ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731251837498 2024-11-10T15:17:19,215 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting b0d725ce8a32454fa5fa895633e45948, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731251837498 2024-11-10T15:17:19,215 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d556a01c60e945708404461b4119c313, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731251838157 2024-11-10T15:17:19,215 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 14037ffb6fb042e7a1871e3414200be0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731251838157 2024-11-10T15:17:19,244 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#B#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:19,244 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/c5470e2881e842eb9b32a218f88a3313 is 50, key is test_row_0/B:col10/1731251838158/Put/seqid=0 2024-11-10T15:17:19,260 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#A#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:19,260 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/c7ca8d5dcc8e4d4b9487c072aa5d4361 is 50, key is test_row_0/A:col10/1731251838158/Put/seqid=0 2024-11-10T15:17:19,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742095_1271 (size=12104) 2024-11-10T15:17:19,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742096_1272 (size=12104) 2024-11-10T15:17:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:19,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T15:17:19,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:19,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:19,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:19,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:19,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:19,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:19,315 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/c7ca8d5dcc8e4d4b9487c072aa5d4361 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/c7ca8d5dcc8e4d4b9487c072aa5d4361 2024-11-10T15:17:19,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/63301fcc19184cc5b4721cf02d92917c is 50, key is test_row_0/A:col10/1731251838731/Put/seqid=0 2024-11-10T15:17:19,325 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/A of c42fcce8c0425e5889d52562aa8da5f6 into c7ca8d5dcc8e4d4b9487c072aa5d4361(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:19,325 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:19,325 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/A, priority=13, startTime=1731251839210; duration=0sec 2024-11-10T15:17:19,325 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:19,325 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:A 2024-11-10T15:17:19,326 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:19,329 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:19,329 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/C is initiating minor compaction (all files) 2024-11-10T15:17:19,329 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/C in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,329 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/14fb2f6a71a841f9ae109e4c026c6c66, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/fce1c3814c14411f8954717fa70a6ba8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/b9f360ea5e6a400ba0d76f9df90cc73a] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=32.9 K 2024-11-10T15:17:19,330 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14fb2f6a71a841f9ae109e4c026c6c66, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731251837443 2024-11-10T15:17:19,331 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fce1c3814c14411f8954717fa70a6ba8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731251837498 2024-11-10T15:17:19,332 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9f360ea5e6a400ba0d76f9df90cc73a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731251838157 2024-11-10T15:17:19,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251899323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251899329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251899334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T15:17:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:19,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251899332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,346 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#C#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:19,347 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/0df113a31c3742048a94b7fcba592f6a is 50, key is test_row_0/C:col10/1731251838158/Put/seqid=0 2024-11-10T15:17:19,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742097_1273 (size=14341) 2024-11-10T15:17:19,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/63301fcc19184cc5b4721cf02d92917c 2024-11-10T15:17:19,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742098_1274 (size=12104) 2024-11-10T15:17:19,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/781dfa37e29745c4aeaa7cbf6b4b1060 is 50, key is test_row_0/B:col10/1731251838731/Put/seqid=0 2024-11-10T15:17:19,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742099_1275 (size=12001) 2024-11-10T15:17:19,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38552 deadline: 1731251899381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251899441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251899451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251899451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251899451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T15:17:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T15:17:19,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:19,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251899652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251899661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251899661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251899661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,695 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/c5470e2881e842eb9b32a218f88a3313 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/c5470e2881e842eb9b32a218f88a3313 2024-11-10T15:17:19,702 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/B of c42fcce8c0425e5889d52562aa8da5f6 into c5470e2881e842eb9b32a218f88a3313(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:19,702 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:19,702 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/B, priority=13, startTime=1731251839211; duration=0sec 2024-11-10T15:17:19,702 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:19,702 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:B 2024-11-10T15:17:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T15:17:19,763 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/0df113a31c3742048a94b7fcba592f6a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/0df113a31c3742048a94b7fcba592f6a 2024-11-10T15:17:19,769 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/C of c42fcce8c0425e5889d52562aa8da5f6 into 0df113a31c3742048a94b7fcba592f6a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:19,769 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:19,769 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/C, priority=13, startTime=1731251839211; duration=0sec 2024-11-10T15:17:19,769 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:19,769 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:C 2024-11-10T15:17:19,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/781dfa37e29745c4aeaa7cbf6b4b1060 2024-11-10T15:17:19,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/39eee88dacc64bcba2f2b94e8e9a5f26 is 50, key is test_row_0/C:col10/1731251838731/Put/seqid=0 2024-11-10T15:17:19,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742100_1276 (size=12001) 2024-11-10T15:17:19,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T15:17:19,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:19,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38552 deadline: 1731251899901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T15:17:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:19,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:19,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251899961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251899971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251899971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251899971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:20,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T15:17:20,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:20,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:20,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:20,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:20,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:20,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/39eee88dacc64bcba2f2b94e8e9a5f26 2024-11-10T15:17:20,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/63301fcc19184cc5b4721cf02d92917c as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/63301fcc19184cc5b4721cf02d92917c 2024-11-10T15:17:20,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/63301fcc19184cc5b4721cf02d92917c, entries=200, sequenceid=80, filesize=14.0 K 2024-11-10T15:17:20,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/781dfa37e29745c4aeaa7cbf6b4b1060 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/781dfa37e29745c4aeaa7cbf6b4b1060 2024-11-10T15:17:20,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/781dfa37e29745c4aeaa7cbf6b4b1060, entries=150, sequenceid=80, filesize=11.7 K 2024-11-10T15:17:20,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/39eee88dacc64bcba2f2b94e8e9a5f26 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/39eee88dacc64bcba2f2b94e8e9a5f26 2024-11-10T15:17:20,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/39eee88dacc64bcba2f2b94e8e9a5f26, entries=150, sequenceid=80, filesize=11.7 K 2024-11-10T15:17:20,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c42fcce8c0425e5889d52562aa8da5f6 in 918ms, sequenceid=80, compaction requested=false 2024-11-10T15:17:20,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:20,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T15:17:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:20,264 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T15:17:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:20,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/5cc037b209ef4000bd0d420857af71e4 is 50, key is test_row_0/A:col10/1731251839329/Put/seqid=0 2024-11-10T15:17:20,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742101_1277 (size=12001) 2024-11-10T15:17:20,292 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/5cc037b209ef4000bd0d420857af71e4 2024-11-10T15:17:20,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/47eea70f486443ef9b4e0e42d36d72b9 is 50, key is test_row_0/B:col10/1731251839329/Put/seqid=0 2024-11-10T15:17:20,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742102_1278 (size=12001) 2024-11-10T15:17:20,316 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/47eea70f486443ef9b4e0e42d36d72b9 2024-11-10T15:17:20,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/5ac3ca9040c44b9682b17e2b10431a7c is 50, key is test_row_0/C:col10/1731251839329/Put/seqid=0 2024-11-10T15:17:20,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742103_1279 (size=12001) 2024-11-10T15:17:20,373 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/5ac3ca9040c44b9682b17e2b10431a7c 2024-11-10T15:17:20,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/5cc037b209ef4000bd0d420857af71e4 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/5cc037b209ef4000bd0d420857af71e4 2024-11-10T15:17:20,391 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/5cc037b209ef4000bd0d420857af71e4, entries=150, sequenceid=94, filesize=11.7 K 2024-11-10T15:17:20,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/47eea70f486443ef9b4e0e42d36d72b9 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/47eea70f486443ef9b4e0e42d36d72b9 2024-11-10T15:17:20,411 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/47eea70f486443ef9b4e0e42d36d72b9, entries=150, sequenceid=94, filesize=11.7 K 2024-11-10T15:17:20,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/5ac3ca9040c44b9682b17e2b10431a7c as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/5ac3ca9040c44b9682b17e2b10431a7c 2024-11-10T15:17:20,423 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/5ac3ca9040c44b9682b17e2b10431a7c, entries=150, sequenceid=94, filesize=11.7 K 2024-11-10T15:17:20,424 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for c42fcce8c0425e5889d52562aa8da5f6 in 160ms, sequenceid=94, compaction requested=true 2024-11-10T15:17:20,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:20,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:20,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-10T15:17:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-10T15:17:20,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-10T15:17:20,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8530 sec 2024-11-10T15:17:20,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.8610 sec 2024-11-10T15:17:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:20,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T15:17:20,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:20,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:20,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:20,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:20,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:20,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:20,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/8544ee26a18047fe87b3e4c6ff429c16 is 50, key is test_row_0/A:col10/1731251840478/Put/seqid=0 2024-11-10T15:17:20,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742104_1280 (size=12001) 2024-11-10T15:17:20,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731251840548, max timestamp=1731251840558Num uniques 859; 2024-11-10T15:17:20,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731251840548, max timestamp=1731251840558Num uniques 859; 2024-11-10T15:17:20,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T15:17:20,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731251840548, max timestamp=1731251840558Num uniques 859; 2024-11-10T15:17:20,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T15:17:20,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T15:17:20,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251900585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:20,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251900587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:20,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:20,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251900589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:20,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251900591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:20,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251900702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:20,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251900702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:20,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251900702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:20,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251900702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T15:17:20,732 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:20,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:17:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-10T15:17:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T15:17:20,738 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:17:20,739 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:17:20,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:17:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T15:17:20,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T15:17:20,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:20,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:20,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:20,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:20,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:20,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/8544ee26a18047fe87b3e4c6ff429c16 2024-11-10T15:17:20,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:20,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251900922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251900922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:20,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:20,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38552 deadline: 1731251900922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:20,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:20,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251900922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:20,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:20,931 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in c42fcce8c0425e5889d52562aa8da5f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251900922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:20,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/64a87411b744422ea5a05e021fba754d is 50, key is test_row_0/B:col10/1731251840478/Put/seqid=0 2024-11-10T15:17:20,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742105_1281 (size=12001) 2024-11-10T15:17:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T15:17:21,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T15:17:21,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:21,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:21,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:21,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,112 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T15:17:21,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T15:17:21,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:21,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:21,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:21,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251901244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:21,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251901244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:21,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251901249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:21,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251901249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:21,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T15:17:21,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:21,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:21,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:21,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T15:17:21,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/64a87411b744422ea5a05e021fba754d 2024-11-10T15:17:21,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/a17c5381927f46f18db8fe725318fdd4 is 50, key is test_row_0/C:col10/1731251840478/Put/seqid=0 2024-11-10T15:17:21,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742106_1282 (size=12001) 2024-11-10T15:17:21,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T15:17:21,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:21,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:21,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:21,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T15:17:21,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:21,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:21,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:21,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251901751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:21,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251901751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:21,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:21,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:21,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:21,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:21,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:21,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:21,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251901761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:21,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251901761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:21,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:21,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:21,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:21,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T15:17:21,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:21,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:21,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:21,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:21,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/a17c5381927f46f18db8fe725318fdd4 2024-11-10T15:17:21,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/8544ee26a18047fe87b3e4c6ff429c16 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/8544ee26a18047fe87b3e4c6ff429c16 2024-11-10T15:17:21,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/8544ee26a18047fe87b3e4c6ff429c16, entries=150, sequenceid=104, filesize=11.7 K 2024-11-10T15:17:21,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/64a87411b744422ea5a05e021fba754d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/64a87411b744422ea5a05e021fba754d 2024-11-10T15:17:21,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/64a87411b744422ea5a05e021fba754d, entries=150, sequenceid=104, filesize=11.7 K 2024-11-10T15:17:21,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/a17c5381927f46f18db8fe725318fdd4 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/a17c5381927f46f18db8fe725318fdd4 2024-11-10T15:17:21,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/a17c5381927f46f18db8fe725318fdd4, entries=150, sequenceid=104, filesize=11.7 K 2024-11-10T15:17:21,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c42fcce8c0425e5889d52562aa8da5f6 in 1377ms, sequenceid=104, compaction requested=true 2024-11-10T15:17:21,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:21,867 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:17:21,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:17:21,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:21,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:17:21,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:21,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:17:21,868 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:17:21,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:21,868 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:17:21,869 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/A is initiating minor compaction (all files) 2024-11-10T15:17:21,869 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/A in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:21,869 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/c7ca8d5dcc8e4d4b9487c072aa5d4361, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/63301fcc19184cc5b4721cf02d92917c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/5cc037b209ef4000bd0d420857af71e4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/8544ee26a18047fe87b3e4c6ff429c16] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=49.3 K 2024-11-10T15:17:21,869 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:17:21,869 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/B is initiating minor compaction (all files) 2024-11-10T15:17:21,869 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/B in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:21,869 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/c5470e2881e842eb9b32a218f88a3313, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/781dfa37e29745c4aeaa7cbf6b4b1060, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/47eea70f486443ef9b4e0e42d36d72b9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/64a87411b744422ea5a05e021fba754d] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=47.0 K 2024-11-10T15:17:21,869 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting c5470e2881e842eb9b32a218f88a3313, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731251838157 2024-11-10T15:17:21,869 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7ca8d5dcc8e4d4b9487c072aa5d4361, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731251838157 2024-11-10T15:17:21,870 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 781dfa37e29745c4aeaa7cbf6b4b1060, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731251838731 2024-11-10T15:17:21,870 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63301fcc19184cc5b4721cf02d92917c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731251838731 2024-11-10T15:17:21,870 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 47eea70f486443ef9b4e0e42d36d72b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731251839326 2024-11-10T15:17:21,870 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cc037b209ef4000bd0d420857af71e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731251839326 2024-11-10T15:17:21,870 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 64a87411b744422ea5a05e021fba754d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731251840473 2024-11-10T15:17:21,871 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8544ee26a18047fe87b3e4c6ff429c16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731251840473 2024-11-10T15:17:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T15:17:21,880 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#B#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:21,881 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/d05a78002409473d9dda01fc8e54fde5 is 50, key is test_row_0/B:col10/1731251840478/Put/seqid=0 2024-11-10T15:17:21,899 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:21,900 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/3223faa581644985b184878cdf967577 is 50, key is test_row_0/A:col10/1731251840478/Put/seqid=0 2024-11-10T15:17:21,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742107_1283 (size=12241) 2024-11-10T15:17:21,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742108_1284 (size=12241) 2024-11-10T15:17:21,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T15:17:21,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:21,990 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-10T15:17:21,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:21,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/08215547116940cba6202fea3bee1043 is 50, key is test_row_0/A:col10/1731251840587/Put/seqid=0 2024-11-10T15:17:22,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742109_1285 (size=12151) 2024-11-10T15:17:22,315 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/d05a78002409473d9dda01fc8e54fde5 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/d05a78002409473d9dda01fc8e54fde5 2024-11-10T15:17:22,319 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/B of c42fcce8c0425e5889d52562aa8da5f6 into d05a78002409473d9dda01fc8e54fde5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:22,319 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:22,319 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/B, priority=12, startTime=1731251841868; duration=0sec 2024-11-10T15:17:22,320 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:22,320 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:B 2024-11-10T15:17:22,320 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:17:22,321 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:17:22,321 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/C is initiating minor compaction (all files) 2024-11-10T15:17:22,321 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/C in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:22,321 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/0df113a31c3742048a94b7fcba592f6a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/39eee88dacc64bcba2f2b94e8e9a5f26, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/5ac3ca9040c44b9682b17e2b10431a7c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/a17c5381927f46f18db8fe725318fdd4] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=47.0 K 2024-11-10T15:17:22,321 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0df113a31c3742048a94b7fcba592f6a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731251838157 2024-11-10T15:17:22,322 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 39eee88dacc64bcba2f2b94e8e9a5f26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731251838731 2024-11-10T15:17:22,322 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ac3ca9040c44b9682b17e2b10431a7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731251839326 2024-11-10T15:17:22,323 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting a17c5381927f46f18db8fe725318fdd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731251840473 2024-11-10T15:17:22,332 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#C#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:22,334 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/987c9a1b530a4a6c8ebbeaf1244589fa is 50, key is test_row_0/C:col10/1731251840478/Put/seqid=0 2024-11-10T15:17:22,341 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/3223faa581644985b184878cdf967577 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/3223faa581644985b184878cdf967577 2024-11-10T15:17:22,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742110_1286 (size=12241) 2024-11-10T15:17:22,348 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/A of c42fcce8c0425e5889d52562aa8da5f6 into 3223faa581644985b184878cdf967577(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:22,348 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:22,348 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/A, priority=12, startTime=1731251841867; duration=0sec 2024-11-10T15:17:22,348 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:22,348 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:A 2024-11-10T15:17:22,350 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/987c9a1b530a4a6c8ebbeaf1244589fa as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/987c9a1b530a4a6c8ebbeaf1244589fa 2024-11-10T15:17:22,355 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/C of c42fcce8c0425e5889d52562aa8da5f6 into 987c9a1b530a4a6c8ebbeaf1244589fa(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:22,355 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:22,355 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/C, priority=12, startTime=1731251841868; duration=0sec 2024-11-10T15:17:22,355 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:22,355 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:C 2024-11-10T15:17:22,418 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/08215547116940cba6202fea3bee1043 2024-11-10T15:17:22,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/35c547115c214d9bb4c45bb17a9c426b is 50, key is test_row_0/B:col10/1731251840587/Put/seqid=0 2024-11-10T15:17:22,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742111_1287 (size=12151) 2024-11-10T15:17:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:22,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:22,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251902778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:22,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:22,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:22,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:22,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251902782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:22,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251902783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:22,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251902783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:22,788 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in c42fcce8c0425e5889d52562aa8da5f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:22,833 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/35c547115c214d9bb4c45bb17a9c426b 2024-11-10T15:17:22,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/b9da804cd8ab4bfda9ee92f9a3447091 is 50, key is test_row_0/C:col10/1731251840587/Put/seqid=0 2024-11-10T15:17:22,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742112_1288 (size=12151) 2024-11-10T15:17:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T15:17:22,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251902891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:22,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251902891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:22,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251902892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:22,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:22,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:22,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:22,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:22,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:22,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:22,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:22,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38552 deadline: 1731251902951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:22,955 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in c42fcce8c0425e5889d52562aa8da5f6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:22,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:22,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:22,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251903101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:23,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251903112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251903112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:23,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:23,250 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/b9da804cd8ab4bfda9ee92f9a3447091 2024-11-10T15:17:23,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/08215547116940cba6202fea3bee1043 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/08215547116940cba6202fea3bee1043 2024-11-10T15:17:23,263 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/08215547116940cba6202fea3bee1043, entries=150, sequenceid=139, filesize=11.9 K 2024-11-10T15:17:23,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/35c547115c214d9bb4c45bb17a9c426b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/35c547115c214d9bb4c45bb17a9c426b 2024-11-10T15:17:23,272 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/35c547115c214d9bb4c45bb17a9c426b, entries=150, sequenceid=139, filesize=11.9 K 2024-11-10T15:17:23,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/b9da804cd8ab4bfda9ee92f9a3447091 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/b9da804cd8ab4bfda9ee92f9a3447091 2024-11-10T15:17:23,284 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/b9da804cd8ab4bfda9ee92f9a3447091, entries=150, sequenceid=139, filesize=11.9 K 2024-11-10T15:17:23,285 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for c42fcce8c0425e5889d52562aa8da5f6 in 1295ms, sequenceid=139, compaction requested=false 2024-11-10T15:17:23,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:23,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:23,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-10T15:17:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-10T15:17:23,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-10T15:17:23,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5480 sec 2024-11-10T15:17:23,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.5530 sec 2024-11-10T15:17:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:23,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T15:17:23,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:23,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:23,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:23,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:23,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:23,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:23,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/716519a7a0d64101a1e8a2ac386713f8 is 50, key is test_row_0/A:col10/1731251842781/Put/seqid=0 2024-11-10T15:17:23,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742113_1289 (size=14541) 2024-11-10T15:17:23,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251903450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:23,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251903451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:23,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:23,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251903452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:23,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251903561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:23,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:23,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251903561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:23,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:23,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:23,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251903561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:23,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251903772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:23,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251903772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251903772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:23,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/716519a7a0d64101a1e8a2ac386713f8 2024-11-10T15:17:23,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/966e185dedcc41df8e2a4327b28b0897 is 50, key is test_row_0/B:col10/1731251842781/Put/seqid=0 2024-11-10T15:17:23,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742114_1290 (size=12151) 2024-11-10T15:17:24,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251904081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:24,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251904081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:24,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251904081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:24,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/966e185dedcc41df8e2a4327b28b0897 2024-11-10T15:17:24,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/f80b7b159c1a4af5a43919f50dccd865 is 50, key is test_row_0/C:col10/1731251842781/Put/seqid=0 2024-11-10T15:17:24,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742115_1291 (size=12151) 2024-11-10T15:17:24,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251904591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:24,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251904591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:24,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:24,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251904591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:24,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/f80b7b159c1a4af5a43919f50dccd865 2024-11-10T15:17:24,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/716519a7a0d64101a1e8a2ac386713f8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/716519a7a0d64101a1e8a2ac386713f8 2024-11-10T15:17:24,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/716519a7a0d64101a1e8a2ac386713f8, entries=200, sequenceid=154, filesize=14.2 K 2024-11-10T15:17:24,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/966e185dedcc41df8e2a4327b28b0897 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/966e185dedcc41df8e2a4327b28b0897 2024-11-10T15:17:24,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/966e185dedcc41df8e2a4327b28b0897, entries=150, sequenceid=154, filesize=11.9 K 2024-11-10T15:17:24,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/f80b7b159c1a4af5a43919f50dccd865 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/f80b7b159c1a4af5a43919f50dccd865 2024-11-10T15:17:24,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/f80b7b159c1a4af5a43919f50dccd865, entries=150, sequenceid=154, filesize=11.9 K 2024-11-10T15:17:24,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c42fcce8c0425e5889d52562aa8da5f6 in 1262ms, sequenceid=154, compaction requested=true 2024-11-10T15:17:24,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:24,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:17:24,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:24,676 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:24,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:17:24,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:24,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:17:24,676 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:24,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:24,677 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:24,677 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:24,677 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/A is initiating minor compaction (all files) 2024-11-10T15:17:24,677 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/B is initiating minor compaction (all files) 2024-11-10T15:17:24,677 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/A in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:24,677 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/3223faa581644985b184878cdf967577, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/08215547116940cba6202fea3bee1043, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/716519a7a0d64101a1e8a2ac386713f8] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=38.0 K 2024-11-10T15:17:24,677 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/B in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:24,677 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/d05a78002409473d9dda01fc8e54fde5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/35c547115c214d9bb4c45bb17a9c426b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/966e185dedcc41df8e2a4327b28b0897] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=35.7 K 2024-11-10T15:17:24,677 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3223faa581644985b184878cdf967577, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731251840473 2024-11-10T15:17:24,677 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08215547116940cba6202fea3bee1043, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731251840576 2024-11-10T15:17:24,677 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting d05a78002409473d9dda01fc8e54fde5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731251840473 2024-11-10T15:17:24,678 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 716519a7a0d64101a1e8a2ac386713f8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731251842775 2024-11-10T15:17:24,678 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 35c547115c214d9bb4c45bb17a9c426b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731251840576 2024-11-10T15:17:24,678 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 966e185dedcc41df8e2a4327b28b0897, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731251842781 2024-11-10T15:17:24,684 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:24,684 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/c002291e73e44a3c96909e354bbbcb27 is 50, key is test_row_0/A:col10/1731251842781/Put/seqid=0 2024-11-10T15:17:24,684 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:24,685 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/41f9bce7d70c49a0a12077390ab8cc4d is 50, key is test_row_0/B:col10/1731251842781/Put/seqid=0 2024-11-10T15:17:24,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742116_1292 (size=12493) 2024-11-10T15:17:24,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742117_1293 (size=12493) 2024-11-10T15:17:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:24,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T15:17:24,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:24,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:24,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:24,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:24,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:24,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:24,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/2b32d537367044f38f2a12cc1d8bab4d is 50, key is test_row_0/A:col10/1731251844802/Put/seqid=0 2024-11-10T15:17:24,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742118_1294 (size=12151) 2024-11-10T15:17:24,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/2b32d537367044f38f2a12cc1d8bab4d 2024-11-10T15:17:24,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/9b019df6c0854791afb50e8dcca5fe0b is 50, key is test_row_0/B:col10/1731251844802/Put/seqid=0 2024-11-10T15:17:24,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742119_1295 (size=12151) 2024-11-10T15:17:24,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251904890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T15:17:24,902 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:24,902 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:17:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-10T15:17:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T15:17:24,904 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:17:24,905 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:17:24,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:17:25,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251905001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T15:17:25,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T15:17:25,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:25,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:25,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:25,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:25,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:25,093 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/c002291e73e44a3c96909e354bbbcb27 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/c002291e73e44a3c96909e354bbbcb27 2024-11-10T15:17:25,095 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/41f9bce7d70c49a0a12077390ab8cc4d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/41f9bce7d70c49a0a12077390ab8cc4d 2024-11-10T15:17:25,097 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/A of c42fcce8c0425e5889d52562aa8da5f6 into c002291e73e44a3c96909e354bbbcb27(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:25,097 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:25,097 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/A, priority=13, startTime=1731251844676; duration=0sec 2024-11-10T15:17:25,097 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:25,097 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:A 2024-11-10T15:17:25,097 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:25,098 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:25,098 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/C is initiating minor compaction (all files) 2024-11-10T15:17:25,098 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/C in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:25,098 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/987c9a1b530a4a6c8ebbeaf1244589fa, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/b9da804cd8ab4bfda9ee92f9a3447091, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/f80b7b159c1a4af5a43919f50dccd865] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=35.7 K 2024-11-10T15:17:25,098 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 987c9a1b530a4a6c8ebbeaf1244589fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731251840473 2024-11-10T15:17:25,099 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9da804cd8ab4bfda9ee92f9a3447091, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731251840576 2024-11-10T15:17:25,099 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f80b7b159c1a4af5a43919f50dccd865, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731251842781 2024-11-10T15:17:25,100 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/B of c42fcce8c0425e5889d52562aa8da5f6 into 41f9bce7d70c49a0a12077390ab8cc4d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:25,100 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:25,100 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/B, priority=13, startTime=1731251844676; duration=0sec 2024-11-10T15:17:25,100 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:25,100 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:B 2024-11-10T15:17:25,105 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#C#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:25,105 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/e79f9b158b3a4fc48959d4773af39a29 is 50, key is test_row_0/C:col10/1731251842781/Put/seqid=0 2024-11-10T15:17:25,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742120_1296 (size=12493) 2024-11-10T15:17:25,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251905211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:25,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T15:17:25,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:25,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:25,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:25,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:25,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T15:17:25,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/9b019df6c0854791afb50e8dcca5fe0b 2024-11-10T15:17:25,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/04d4a832e1c14fb0aa827b157e13038a is 50, key is test_row_0/C:col10/1731251844802/Put/seqid=0 2024-11-10T15:17:25,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742121_1297 (size=12151) 2024-11-10T15:17:25,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T15:17:25,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:25,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:25,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:25,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:25,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:25,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:25,513 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/e79f9b158b3a4fc48959d4773af39a29 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/e79f9b158b3a4fc48959d4773af39a29 2024-11-10T15:17:25,517 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/C of c42fcce8c0425e5889d52562aa8da5f6 into e79f9b158b3a4fc48959d4773af39a29(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:25,517 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:25,517 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/C, priority=13, startTime=1731251844676; duration=0sec 2024-11-10T15:17:25,517 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:25,517 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:C 2024-11-10T15:17:25,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T15:17:25,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:25,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:25,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:25,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:25,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:25,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:25,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251905521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:25,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T15:17:25,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251905601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:25,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251905601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:25,602 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_1' in c42fcce8c0425e5889d52562aa8da5f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:25,603 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in c42fcce8c0425e5889d52562aa8da5f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:25,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251905614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:25,617 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_2' in c42fcce8c0425e5889d52562aa8da5f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:25,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/04d4a832e1c14fb0aa827b157e13038a 2024-11-10T15:17:25,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/2b32d537367044f38f2a12cc1d8bab4d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/2b32d537367044f38f2a12cc1d8bab4d 2024-11-10T15:17:25,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/2b32d537367044f38f2a12cc1d8bab4d, entries=150, sequenceid=175, filesize=11.9 K 2024-11-10T15:17:25,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/9b019df6c0854791afb50e8dcca5fe0b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/9b019df6c0854791afb50e8dcca5fe0b 2024-11-10T15:17:25,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/9b019df6c0854791afb50e8dcca5fe0b, entries=150, sequenceid=175, filesize=11.9 K 2024-11-10T15:17:25,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/04d4a832e1c14fb0aa827b157e13038a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/04d4a832e1c14fb0aa827b157e13038a 2024-11-10T15:17:25,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/04d4a832e1c14fb0aa827b157e13038a, entries=150, sequenceid=175, filesize=11.9 K 2024-11-10T15:17:25,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T15:17:25,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c42fcce8c0425e5889d52562aa8da5f6 in 867ms, sequenceid=175, compaction requested=false 2024-11-10T15:17:25,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:25,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:25,671 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T15:17:25,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:25,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:25,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:25,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:25,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:25,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:25,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/d66f6037326444d9a247bb4f9debde5b is 50, key is test_row_0/A:col10/1731251844834/Put/seqid=0 2024-11-10T15:17:25,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742122_1298 (size=12151) 2024-11-10T15:17:25,679 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/d66f6037326444d9a247bb4f9debde5b 2024-11-10T15:17:25,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/f81a9bc4f3b4474bb661971c16cacee2 is 50, key is test_row_0/B:col10/1731251844834/Put/seqid=0 2024-11-10T15:17:25,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742123_1299 (size=12151) 2024-11-10T15:17:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:26,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T15:17:26,088 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/f81a9bc4f3b4474bb661971c16cacee2 2024-11-10T15:17:26,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/09a8ab3100f042a7a067f30d1480b552 is 50, key is test_row_0/C:col10/1731251844834/Put/seqid=0 2024-11-10T15:17:26,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742124_1300 (size=12151) 2024-11-10T15:17:26,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251846103, max timestamp=1731251846132Num uniques 859; 2024-11-10T15:17:26,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:17:26,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251846103, max timestamp=1731251846132Num uniques 859; 2024-11-10T15:17:26,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251846103, max timestamp=1731251846132Num uniques 859; 2024-11-10T15:17:26,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:17:26,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:17:26,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251906218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:26,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251906331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:26,522 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/09a8ab3100f042a7a067f30d1480b552 2024-11-10T15:17:26,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/d66f6037326444d9a247bb4f9debde5b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/d66f6037326444d9a247bb4f9debde5b 2024-11-10T15:17:26,536 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/d66f6037326444d9a247bb4f9debde5b, entries=150, sequenceid=189, filesize=11.9 K 2024-11-10T15:17:26,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/f81a9bc4f3b4474bb661971c16cacee2 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/f81a9bc4f3b4474bb661971c16cacee2 2024-11-10T15:17:26,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251906542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:26,550 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/f81a9bc4f3b4474bb661971c16cacee2, entries=150, sequenceid=189, filesize=11.9 K 2024-11-10T15:17:26,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/09a8ab3100f042a7a067f30d1480b552 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/09a8ab3100f042a7a067f30d1480b552 2024-11-10T15:17:26,559 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/09a8ab3100f042a7a067f30d1480b552, entries=150, sequenceid=189, filesize=11.9 K 2024-11-10T15:17:26,560 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c42fcce8c0425e5889d52562aa8da5f6 in 889ms, sequenceid=189, compaction requested=true 2024-11-10T15:17:26,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:26,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:26,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-10T15:17:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-10T15:17:26,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-10T15:17:26,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6570 sec 2024-11-10T15:17:26,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.6680 sec 2024-11-10T15:17:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:26,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-10T15:17:26,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:26,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:26,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:26,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:26,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:26,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:26,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/e08017fbb0fb4430a6dcca7ab058a330 is 50, key is test_row_0/A:col10/1731251846179/Put/seqid=0 2024-11-10T15:17:26,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742125_1301 (size=14541) 2024-11-10T15:17:26,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251906897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:26,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38552 deadline: 1731251906991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:26,995 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in c42fcce8c0425e5889d52562aa8da5f6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:26,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:27,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251907011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:27,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:27,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T15:17:27,051 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:27,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:17:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-10T15:17:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T15:17:27,054 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:17:27,054 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:17:27,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:17:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T15:17:27,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T15:17:27,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:27,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:27,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:27,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251907221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:27,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/e08017fbb0fb4430a6dcca7ab058a330 2024-11-10T15:17:27,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/1a79edea1c0e42989b052604760a4e68 is 50, key is test_row_0/B:col10/1731251846179/Put/seqid=0 2024-11-10T15:17:27,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742126_1302 (size=12151) 2024-11-10T15:17:27,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T15:17:27,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:27,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:27,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:27,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T15:17:27,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T15:17:27,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:27,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:27,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:27,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251907532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:27,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:27,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:27,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:27,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251907622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:27,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251907622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:27,625 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_1' in c42fcce8c0425e5889d52562aa8da5f6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:27,625 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in c42fcce8c0425e5889d52562aa8da5f6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:27,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251907631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:27,634 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_2' in c42fcce8c0425e5889d52562aa8da5f6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:27,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:27,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:27,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:27,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T15:17:27,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:27,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:27,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:27,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T15:17:27,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/1a79edea1c0e42989b052604760a4e68 2024-11-10T15:17:27,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/d42417e3a7c14f0f9593f5011b7ccda4 is 50, key is test_row_0/C:col10/1731251846179/Put/seqid=0 2024-11-10T15:17:27,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742127_1303 (size=12151) 2024-11-10T15:17:27,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T15:17:27,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:27,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:27,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:27,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T15:17:27,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:27,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:27,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:27,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:28,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251908042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:28,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/d42417e3a7c14f0f9593f5011b7ccda4 2024-11-10T15:17:28,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/e08017fbb0fb4430a6dcca7ab058a330 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e08017fbb0fb4430a6dcca7ab058a330 2024-11-10T15:17:28,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e08017fbb0fb4430a6dcca7ab058a330, entries=200, sequenceid=226, filesize=14.2 K 2024-11-10T15:17:28,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/1a79edea1c0e42989b052604760a4e68 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/1a79edea1c0e42989b052604760a4e68 2024-11-10T15:17:28,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/1a79edea1c0e42989b052604760a4e68, entries=150, sequenceid=226, filesize=11.9 K 2024-11-10T15:17:28,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/d42417e3a7c14f0f9593f5011b7ccda4 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/d42417e3a7c14f0f9593f5011b7ccda4 2024-11-10T15:17:28,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T15:17:28,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:28,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:28,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:28,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:28,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:28,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/d42417e3a7c14f0f9593f5011b7ccda4, entries=150, sequenceid=226, filesize=11.9 K 2024-11-10T15:17:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:28,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for c42fcce8c0425e5889d52562aa8da5f6 in 1279ms, sequenceid=226, compaction requested=true 2024-11-10T15:17:28,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:28,133 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:17:28,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:17:28,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:28,133 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:17:28,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:17:28,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:28,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:17:28,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:28,134 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:17:28,134 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51336 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:17:28,134 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/B is initiating minor compaction (all files) 2024-11-10T15:17:28,134 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/A is initiating minor compaction (all files) 2024-11-10T15:17:28,134 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/B in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:28,134 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/A in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:28,135 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/41f9bce7d70c49a0a12077390ab8cc4d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/9b019df6c0854791afb50e8dcca5fe0b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/f81a9bc4f3b4474bb661971c16cacee2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/1a79edea1c0e42989b052604760a4e68] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=47.8 K 2024-11-10T15:17:28,135 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/c002291e73e44a3c96909e354bbbcb27, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/2b32d537367044f38f2a12cc1d8bab4d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/d66f6037326444d9a247bb4f9debde5b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e08017fbb0fb4430a6dcca7ab058a330] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=50.1 K 2024-11-10T15:17:28,135 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 41f9bce7d70c49a0a12077390ab8cc4d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731251842781 2024-11-10T15:17:28,135 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c002291e73e44a3c96909e354bbbcb27, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731251842781 2024-11-10T15:17:28,135 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b019df6c0854791afb50e8dcca5fe0b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731251843438 2024-11-10T15:17:28,135 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b32d537367044f38f2a12cc1d8bab4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731251843438 2024-11-10T15:17:28,135 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting f81a9bc4f3b4474bb661971c16cacee2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731251844834 2024-11-10T15:17:28,135 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d66f6037326444d9a247bb4f9debde5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731251844834 2024-11-10T15:17:28,136 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a79edea1c0e42989b052604760a4e68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731251846179 2024-11-10T15:17:28,136 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e08017fbb0fb4430a6dcca7ab058a330, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731251846168 2024-11-10T15:17:28,143 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#B#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:28,143 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/8362f557341a4b79932873d843f73962 is 50, key is test_row_0/B:col10/1731251846179/Put/seqid=0 2024-11-10T15:17:28,145 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#A#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:28,146 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/e4ce55cf6b9141e4ab7a899ef8bb4033 is 50, key is test_row_0/A:col10/1731251846179/Put/seqid=0 2024-11-10T15:17:28,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742128_1304 (size=12629) 2024-11-10T15:17:28,153 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/8362f557341a4b79932873d843f73962 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/8362f557341a4b79932873d843f73962 2024-11-10T15:17:28,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742129_1305 (size=12629) 2024-11-10T15:17:28,158 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/B of c42fcce8c0425e5889d52562aa8da5f6 into 8362f557341a4b79932873d843f73962(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:28,158 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:28,159 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/B, priority=12, startTime=1731251848133; duration=0sec 2024-11-10T15:17:28,159 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:28,159 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:B 2024-11-10T15:17:28,159 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:17:28,160 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:17:28,161 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/C is initiating minor compaction (all files) 2024-11-10T15:17:28,161 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/C in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:28,161 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/e79f9b158b3a4fc48959d4773af39a29, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/04d4a832e1c14fb0aa827b157e13038a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/09a8ab3100f042a7a067f30d1480b552, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/d42417e3a7c14f0f9593f5011b7ccda4] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=47.8 K 2024-11-10T15:17:28,161 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting e79f9b158b3a4fc48959d4773af39a29, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731251842781 2024-11-10T15:17:28,162 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 04d4a832e1c14fb0aa827b157e13038a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731251843438 2024-11-10T15:17:28,162 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 09a8ab3100f042a7a067f30d1480b552, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731251844834 2024-11-10T15:17:28,163 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting d42417e3a7c14f0f9593f5011b7ccda4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731251846179 2024-11-10T15:17:28,163 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/e4ce55cf6b9141e4ab7a899ef8bb4033 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e4ce55cf6b9141e4ab7a899ef8bb4033 2024-11-10T15:17:28,169 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/A of c42fcce8c0425e5889d52562aa8da5f6 into e4ce55cf6b9141e4ab7a899ef8bb4033(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:28,169 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:28,169 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/A, priority=12, startTime=1731251848133; duration=0sec 2024-11-10T15:17:28,169 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:28,170 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:A 2024-11-10T15:17:28,183 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#C#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:28,183 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/bf2471d1f2a94e3596f13023c1c1c63f is 50, key is test_row_0/C:col10/1731251846179/Put/seqid=0 2024-11-10T15:17:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T15:17:28,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742130_1306 (size=12629) 2024-11-10T15:17:28,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T15:17:28,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:28,284 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T15:17:28,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:28,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:28,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:28,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:28,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:28,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:28,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/f55d924f82dc43e18d194c64d6e52789 is 50, key is test_row_0/A:col10/1731251846895/Put/seqid=0 2024-11-10T15:17:28,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742131_1307 (size=12151) 2024-11-10T15:17:28,294 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/f55d924f82dc43e18d194c64d6e52789 2024-11-10T15:17:28,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/7cc0a18d3da746189a9aa6937d88e582 is 50, key is test_row_0/B:col10/1731251846895/Put/seqid=0 2024-11-10T15:17:28,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742132_1308 (size=12151) 2024-11-10T15:17:28,599 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/bf2471d1f2a94e3596f13023c1c1c63f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/bf2471d1f2a94e3596f13023c1c1c63f 2024-11-10T15:17:28,604 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/C of c42fcce8c0425e5889d52562aa8da5f6 into bf2471d1f2a94e3596f13023c1c1c63f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:28,604 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:28,604 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/C, priority=12, startTime=1731251848134; duration=0sec 2024-11-10T15:17:28,604 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:28,604 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:C 2024-11-10T15:17:28,704 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/7cc0a18d3da746189a9aa6937d88e582 2024-11-10T15:17:28,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/210620fd1e1e447c8dbb078142d3a5cb is 50, key is test_row_0/C:col10/1731251846895/Put/seqid=0 2024-11-10T15:17:28,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742133_1309 (size=12151) 2024-11-10T15:17:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:29,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:29,117 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/210620fd1e1e447c8dbb078142d3a5cb 2024-11-10T15:17:29,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/f55d924f82dc43e18d194c64d6e52789 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/f55d924f82dc43e18d194c64d6e52789 2024-11-10T15:17:29,124 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/f55d924f82dc43e18d194c64d6e52789, entries=150, sequenceid=238, filesize=11.9 K 2024-11-10T15:17:29,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/7cc0a18d3da746189a9aa6937d88e582 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/7cc0a18d3da746189a9aa6937d88e582 2024-11-10T15:17:29,129 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/7cc0a18d3da746189a9aa6937d88e582, entries=150, sequenceid=238, filesize=11.9 K 2024-11-10T15:17:29,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/210620fd1e1e447c8dbb078142d3a5cb as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/210620fd1e1e447c8dbb078142d3a5cb 2024-11-10T15:17:29,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251849079, max timestamp=1731251849127Num uniques 859; 2024-11-10T15:17:29,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251849079, max timestamp=1731251849127Num uniques 859; 2024-11-10T15:17:29,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:17:29,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:17:29,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251849079, max timestamp=1731251849127Num uniques 859; 2024-11-10T15:17:29,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:17:29,135 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/210620fd1e1e447c8dbb078142d3a5cb, entries=150, sequenceid=238, filesize=11.9 K 2024-11-10T15:17:29,136 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=33.54 KB/34350 for c42fcce8c0425e5889d52562aa8da5f6 in 852ms, sequenceid=238, compaction requested=false 2024-11-10T15:17:29,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:29,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:29,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-10T15:17:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-10T15:17:29,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-10T15:17:29,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0830 sec 2024-11-10T15:17:29,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.0880 sec 2024-11-10T15:17:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:29,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-10T15:17:29,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:29,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:29,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:29,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:29,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:29,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:29,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/70918db59c78473aa6242195fe4892a5 is 50, key is test_row_0/A:col10/1731251849133/Put/seqid=0 2024-11-10T15:17:29,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742134_1310 (size=14741) 2024-11-10T15:17:29,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/70918db59c78473aa6242195fe4892a5 2024-11-10T15:17:29,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/e1f48aa8a2cf4500aee987b391e97d6e is 50, key is test_row_0/B:col10/1731251849133/Put/seqid=0 2024-11-10T15:17:29,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742135_1311 (size=12301) 2024-11-10T15:17:29,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/e1f48aa8a2cf4500aee987b391e97d6e 2024-11-10T15:17:29,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/04ed0ea2fab64f3f96c163775d56928a is 50, key is test_row_0/C:col10/1731251849133/Put/seqid=0 2024-11-10T15:17:29,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742136_1312 (size=12301) 2024-11-10T15:17:29,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/04ed0ea2fab64f3f96c163775d56928a 2024-11-10T15:17:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T15:17:29,202 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:29,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:17:29,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/70918db59c78473aa6242195fe4892a5 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/70918db59c78473aa6242195fe4892a5 2024-11-10T15:17:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-10T15:17:29,204 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:17:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T15:17:29,205 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:17:29,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:17:29,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/70918db59c78473aa6242195fe4892a5, entries=200, sequenceid=264, filesize=14.4 K 2024-11-10T15:17:29,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/e1f48aa8a2cf4500aee987b391e97d6e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/e1f48aa8a2cf4500aee987b391e97d6e 2024-11-10T15:17:29,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/e1f48aa8a2cf4500aee987b391e97d6e, entries=150, sequenceid=264, filesize=12.0 K 2024-11-10T15:17:29,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/04ed0ea2fab64f3f96c163775d56928a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/04ed0ea2fab64f3f96c163775d56928a 2024-11-10T15:17:29,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/04ed0ea2fab64f3f96c163775d56928a, entries=150, sequenceid=264, filesize=12.0 K 2024-11-10T15:17:29,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=80.51 KB/82440 for c42fcce8c0425e5889d52562aa8da5f6 in 76ms, sequenceid=264, compaction requested=true 2024-11-10T15:17:29,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:29,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:17:29,220 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:29,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:29,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:17:29,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:29,220 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:29,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:17:29,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:29,221 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:29,221 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/B is initiating minor compaction (all files) 2024-11-10T15:17:29,221 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/B in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:29,221 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/8362f557341a4b79932873d843f73962, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/7cc0a18d3da746189a9aa6937d88e582, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/e1f48aa8a2cf4500aee987b391e97d6e] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=36.2 K 2024-11-10T15:17:29,222 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:29,222 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/A is initiating minor compaction (all files) 2024-11-10T15:17:29,222 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/A in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:29,222 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e4ce55cf6b9141e4ab7a899ef8bb4033, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/f55d924f82dc43e18d194c64d6e52789, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/70918db59c78473aa6242195fe4892a5] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=38.6 K 2024-11-10T15:17:29,223 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 8362f557341a4b79932873d843f73962, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731251846179 2024-11-10T15:17:29,223 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4ce55cf6b9141e4ab7a899ef8bb4033, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731251846179 2024-11-10T15:17:29,223 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f55d924f82dc43e18d194c64d6e52789, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731251846865 2024-11-10T15:17:29,223 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cc0a18d3da746189a9aa6937d88e582, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731251846865 2024-11-10T15:17:29,223 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70918db59c78473aa6242195fe4892a5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731251849133 2024-11-10T15:17:29,223 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting e1f48aa8a2cf4500aee987b391e97d6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731251849133 2024-11-10T15:17:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:29,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T15:17:29,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:29,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:29,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:29,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:29,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:29,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:29,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/7d2a3c71e9674bc89fb15417c133f026 is 50, key is test_row_0/A:col10/1731251849218/Put/seqid=0 2024-11-10T15:17:29,238 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#B#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:29,238 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/18e2fedeb49049ef8338c4aaec681bb8 is 50, key is test_row_0/B:col10/1731251849133/Put/seqid=0 2024-11-10T15:17:29,241 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#A#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:29,241 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/4b7ddda1bd3447bfb6bda5b23f74d6fb is 50, key is test_row_0/A:col10/1731251849133/Put/seqid=0 2024-11-10T15:17:29,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742137_1313 (size=14741) 2024-11-10T15:17:29,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742138_1314 (size=12881) 2024-11-10T15:17:29,269 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/18e2fedeb49049ef8338c4aaec681bb8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/18e2fedeb49049ef8338c4aaec681bb8 2024-11-10T15:17:29,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742139_1315 (size=12881) 2024-11-10T15:17:29,276 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/B of c42fcce8c0425e5889d52562aa8da5f6 into 18e2fedeb49049ef8338c4aaec681bb8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:29,276 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:29,276 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/B, priority=13, startTime=1731251849220; duration=0sec 2024-11-10T15:17:29,276 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:29,276 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:B 2024-11-10T15:17:29,276 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:29,278 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:29,278 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/C is initiating minor compaction (all files) 2024-11-10T15:17:29,278 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/C in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:29,278 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/bf2471d1f2a94e3596f13023c1c1c63f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/210620fd1e1e447c8dbb078142d3a5cb, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/04ed0ea2fab64f3f96c163775d56928a] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=36.2 K 2024-11-10T15:17:29,278 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting bf2471d1f2a94e3596f13023c1c1c63f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731251846179 2024-11-10T15:17:29,279 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/4b7ddda1bd3447bfb6bda5b23f74d6fb as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/4b7ddda1bd3447bfb6bda5b23f74d6fb 2024-11-10T15:17:29,279 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 210620fd1e1e447c8dbb078142d3a5cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731251846865 2024-11-10T15:17:29,279 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 04ed0ea2fab64f3f96c163775d56928a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731251849133 2024-11-10T15:17:29,285 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/A of c42fcce8c0425e5889d52562aa8da5f6 into 4b7ddda1bd3447bfb6bda5b23f74d6fb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:29,285 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:29,285 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/A, priority=13, startTime=1731251849220; duration=0sec 2024-11-10T15:17:29,285 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:29,285 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:A 2024-11-10T15:17:29,290 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#C#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:29,291 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/4010911c41cd49878afe16fff097faf8 is 50, key is test_row_0/C:col10/1731251849133/Put/seqid=0 2024-11-10T15:17:29,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742140_1316 (size=12881) 2024-11-10T15:17:29,303 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/4010911c41cd49878afe16fff097faf8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/4010911c41cd49878afe16fff097faf8 2024-11-10T15:17:29,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251909304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:29,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:29,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:29,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T15:17:29,315 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/C of c42fcce8c0425e5889d52562aa8da5f6 into 4010911c41cd49878afe16fff097faf8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:29,315 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:29,315 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/C, priority=13, startTime=1731251849220; duration=0sec 2024-11-10T15:17:29,316 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:29,316 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:C 2024-11-10T15:17:29,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T15:17:29,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:29,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:29,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:29,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:29,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:29,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251909411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:29,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T15:17:29,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:29,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:29,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:29,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:29,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T15:17:29,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251909621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:29,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/7d2a3c71e9674bc89fb15417c133f026 2024-11-10T15:17:29,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/e95423041084484fa3f9be73af830f3a is 50, key is test_row_0/B:col10/1731251849218/Put/seqid=0 2024-11-10T15:17:29,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742141_1317 (size=12301) 2024-11-10T15:17:29,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T15:17:29,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:29,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:29,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:29,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:29,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:29,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T15:17:29,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:29,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:29,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:29,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:29,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T15:17:29,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251909931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:29,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:29,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:29,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:29,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T15:17:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:29,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:30,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/e95423041084484fa3f9be73af830f3a 2024-11-10T15:17:30,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/6d0cc7f24de945618f961404497ce924 is 50, key is test_row_0/C:col10/1731251849218/Put/seqid=0 2024-11-10T15:17:30,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742142_1318 (size=12301) 2024-11-10T15:17:30,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/6d0cc7f24de945618f961404497ce924 2024-11-10T15:17:30,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/7d2a3c71e9674bc89fb15417c133f026 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/7d2a3c71e9674bc89fb15417c133f026 2024-11-10T15:17:30,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/7d2a3c71e9674bc89fb15417c133f026, entries=200, sequenceid=281, filesize=14.4 K 2024-11-10T15:17:30,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/e95423041084484fa3f9be73af830f3a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/e95423041084484fa3f9be73af830f3a 2024-11-10T15:17:30,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/e95423041084484fa3f9be73af830f3a, entries=150, sequenceid=281, filesize=12.0 K 2024-11-10T15:17:30,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/6d0cc7f24de945618f961404497ce924 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/6d0cc7f24de945618f961404497ce924 2024-11-10T15:17:30,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/6d0cc7f24de945618f961404497ce924, entries=150, sequenceid=281, filesize=12.0 K 2024-11-10T15:17:30,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c42fcce8c0425e5889d52562aa8da5f6 in 871ms, sequenceid=281, compaction requested=false 2024-11-10T15:17:30,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:30,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T15:17:30,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:30,122 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T15:17:30,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:30,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:30,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:30,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:30,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:30,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:30,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/4bad4de814d14d39a42e19dcd36df95d is 50, key is test_row_0/A:col10/1731251849302/Put/seqid=0 2024-11-10T15:17:30,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742143_1319 (size=12301) 2024-11-10T15:17:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T15:17:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:30,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:30,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:30,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251910519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:30,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:30,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:30,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:30,530 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/4bad4de814d14d39a42e19dcd36df95d 2024-11-10T15:17:30,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/01369bb61fec45c1a934142f454fefc6 is 50, key is test_row_0/B:col10/1731251849302/Put/seqid=0 2024-11-10T15:17:30,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742144_1320 (size=12301) 2024-11-10T15:17:30,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251910631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:30,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251910850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:30,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:30,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:30,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:30,942 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/01369bb61fec45c1a934142f454fefc6 2024-11-10T15:17:30,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/0451e88d49f741c4a3e0aebee36a8581 is 50, key is test_row_0/C:col10/1731251849302/Put/seqid=0 2024-11-10T15:17:30,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742145_1321 (size=12301) 2024-11-10T15:17:31,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251911161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T15:17:31,353 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/0451e88d49f741c4a3e0aebee36a8581 2024-11-10T15:17:31,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/4bad4de814d14d39a42e19dcd36df95d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/4bad4de814d14d39a42e19dcd36df95d 2024-11-10T15:17:31,361 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/4bad4de814d14d39a42e19dcd36df95d, entries=150, sequenceid=299, filesize=12.0 K 2024-11-10T15:17:31,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/01369bb61fec45c1a934142f454fefc6 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/01369bb61fec45c1a934142f454fefc6 2024-11-10T15:17:31,367 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/01369bb61fec45c1a934142f454fefc6, entries=150, sequenceid=299, filesize=12.0 K 2024-11-10T15:17:31,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/0451e88d49f741c4a3e0aebee36a8581 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/0451e88d49f741c4a3e0aebee36a8581 2024-11-10T15:17:31,373 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/0451e88d49f741c4a3e0aebee36a8581, entries=150, sequenceid=299, filesize=12.0 K 2024-11-10T15:17:31,374 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c42fcce8c0425e5889d52562aa8da5f6 in 1252ms, sequenceid=299, compaction requested=true 2024-11-10T15:17:31,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:31,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:31,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-10T15:17:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-10T15:17:31,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-10T15:17:31,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1700 sec 2024-11-10T15:17:31,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.1750 sec 2024-11-10T15:17:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:31,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T15:17:31,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:31,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:31,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:31,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:31,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:31,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:31,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/413e860c62c14b3bb358357c493220b7 is 50, key is test_row_0/A:col10/1731251851633/Put/seqid=0 2024-11-10T15:17:31,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742146_1322 (size=14741) 2024-11-10T15:17:31,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251911668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:31,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251911671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38580 deadline: 1731251911672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:31,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251911672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:31,676 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_1' in c42fcce8c0425e5889d52562aa8da5f6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:31,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251911782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:31,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:31,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251911782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:31,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:31,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:31,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251911991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:31,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251911991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:32,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/413e860c62c14b3bb358357c493220b7 2024-11-10T15:17:32,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/c9f18c1045ab4176b3cf0257d3cce212 is 50, key is test_row_0/B:col10/1731251851633/Put/seqid=0 2024-11-10T15:17:32,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742147_1323 (size=12301) 2024-11-10T15:17:32,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/c9f18c1045ab4176b3cf0257d3cce212 2024-11-10T15:17:32,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/44267dda381245c195b217e08b810111 is 50, key is test_row_0/C:col10/1731251851633/Put/seqid=0 2024-11-10T15:17:32,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742148_1324 (size=12301) 2024-11-10T15:17:32,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251912302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:32,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:32,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251912302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:32,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/44267dda381245c195b217e08b810111 2024-11-10T15:17:32,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/413e860c62c14b3bb358357c493220b7 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/413e860c62c14b3bb358357c493220b7 2024-11-10T15:17:32,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/413e860c62c14b3bb358357c493220b7, entries=200, sequenceid=317, filesize=14.4 K 2024-11-10T15:17:32,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/c9f18c1045ab4176b3cf0257d3cce212 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/c9f18c1045ab4176b3cf0257d3cce212 2024-11-10T15:17:32,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/c9f18c1045ab4176b3cf0257d3cce212, entries=150, sequenceid=317, filesize=12.0 K 2024-11-10T15:17:32,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/44267dda381245c195b217e08b810111 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/44267dda381245c195b217e08b810111 2024-11-10T15:17:32,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/44267dda381245c195b217e08b810111, entries=150, sequenceid=317, filesize=12.0 K 2024-11-10T15:17:32,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c42fcce8c0425e5889d52562aa8da5f6 in 855ms, sequenceid=317, compaction requested=true 2024-11-10T15:17:32,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:32,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:17:32,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:32,489 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:17:32,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:17:32,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:32,489 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:17:32,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:17:32,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:32,491 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:17:32,491 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54664 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:17:32,491 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/A is initiating minor compaction (all files) 2024-11-10T15:17:32,491 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/B is initiating minor compaction (all files) 2024-11-10T15:17:32,491 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/A in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:32,491 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/B in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:32,491 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/4b7ddda1bd3447bfb6bda5b23f74d6fb, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/7d2a3c71e9674bc89fb15417c133f026, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/4bad4de814d14d39a42e19dcd36df95d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/413e860c62c14b3bb358357c493220b7] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=53.4 K 2024-11-10T15:17:32,491 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/18e2fedeb49049ef8338c4aaec681bb8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/e95423041084484fa3f9be73af830f3a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/01369bb61fec45c1a934142f454fefc6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/c9f18c1045ab4176b3cf0257d3cce212] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=48.6 K 2024-11-10T15:17:32,491 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b7ddda1bd3447bfb6bda5b23f74d6fb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731251849133 2024-11-10T15:17:32,491 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 18e2fedeb49049ef8338c4aaec681bb8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731251849133 2024-11-10T15:17:32,492 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting e95423041084484fa3f9be73af830f3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731251849185 2024-11-10T15:17:32,492 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d2a3c71e9674bc89fb15417c133f026, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731251849185 2024-11-10T15:17:32,492 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 01369bb61fec45c1a934142f454fefc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731251849252 2024-11-10T15:17:32,492 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bad4de814d14d39a42e19dcd36df95d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731251849252 2024-11-10T15:17:32,492 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting c9f18c1045ab4176b3cf0257d3cce212, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731251850466 2024-11-10T15:17:32,492 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 413e860c62c14b3bb358357c493220b7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731251850466 2024-11-10T15:17:32,501 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#A#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:32,502 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/42dbd47e5d2740c3947966751db8a9a5 is 50, key is test_row_0/A:col10/1731251851633/Put/seqid=0 2024-11-10T15:17:32,502 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#B#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:32,502 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/b96ee92d812444ca9fe828b994722be2 is 50, key is test_row_0/B:col10/1731251851633/Put/seqid=0 2024-11-10T15:17:32,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742149_1325 (size=13017) 2024-11-10T15:17:32,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742150_1326 (size=13017) 2024-11-10T15:17:32,517 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/b96ee92d812444ca9fe828b994722be2 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/b96ee92d812444ca9fe828b994722be2 2024-11-10T15:17:32,518 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/42dbd47e5d2740c3947966751db8a9a5 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/42dbd47e5d2740c3947966751db8a9a5 2024-11-10T15:17:32,526 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/A of c42fcce8c0425e5889d52562aa8da5f6 into 42dbd47e5d2740c3947966751db8a9a5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:32,526 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:32,526 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/B of c42fcce8c0425e5889d52562aa8da5f6 into b96ee92d812444ca9fe828b994722be2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:32,526 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/A, priority=12, startTime=1731251852489; duration=0sec 2024-11-10T15:17:32,527 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:32,527 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/B, priority=12, startTime=1731251852489; duration=0sec 2024-11-10T15:17:32,527 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:32,527 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:32,527 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:A 2024-11-10T15:17:32,527 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:B 2024-11-10T15:17:32,527 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:17:32,529 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:17:32,529 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/C is initiating minor compaction (all files) 2024-11-10T15:17:32,529 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/C in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:32,529 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/4010911c41cd49878afe16fff097faf8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/6d0cc7f24de945618f961404497ce924, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/0451e88d49f741c4a3e0aebee36a8581, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/44267dda381245c195b217e08b810111] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=48.6 K 2024-11-10T15:17:32,530 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4010911c41cd49878afe16fff097faf8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731251849133 2024-11-10T15:17:32,530 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d0cc7f24de945618f961404497ce924, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731251849185 2024-11-10T15:17:32,531 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0451e88d49f741c4a3e0aebee36a8581, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731251849252 2024-11-10T15:17:32,531 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44267dda381245c195b217e08b810111, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731251850466 2024-11-10T15:17:32,541 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#C#compaction#272 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:32,542 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/484f0835dbc24c75a7164c6bdf151c69 is 50, key is test_row_0/C:col10/1731251851633/Put/seqid=0 2024-11-10T15:17:32,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742151_1327 (size=13017) 2024-11-10T15:17:32,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:32,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T15:17:32,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:32,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:32,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:32,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:32,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:32,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:32,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/83f3b33bb0804279b0d97c11db7e1aac is 50, key is test_row_0/A:col10/1731251851669/Put/seqid=0 2024-11-10T15:17:32,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742152_1328 (size=14741) 2024-11-10T15:17:32,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251912753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:32,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:32,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:32,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:32,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251912812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:32,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251912821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:32,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:32,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:32,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:32,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251912861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:32,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:32,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:32,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:32,983 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/484f0835dbc24c75a7164c6bdf151c69 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/484f0835dbc24c75a7164c6bdf151c69 2024-11-10T15:17:32,991 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/C of c42fcce8c0425e5889d52562aa8da5f6 into 484f0835dbc24c75a7164c6bdf151c69(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:32,992 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:32,992 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/C, priority=12, startTime=1731251852489; duration=0sec 2024-11-10T15:17:32,992 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:32,992 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:C 2024-11-10T15:17:33,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251913071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:33,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:33,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:33,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:33,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/83f3b33bb0804279b0d97c11db7e1aac 2024-11-10T15:17:33,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/4ac577cfb2f24d44b1dc237c4b9f63d3 is 50, key is test_row_0/B:col10/1731251851669/Put/seqid=0 2024-11-10T15:17:33,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742153_1329 (size=12301) 2024-11-10T15:17:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T15:17:33,361 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:33,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:17:33,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-10T15:17:33,363 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:17:33,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T15:17:33,364 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:17:33,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:17:33,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:33,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251913391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:33,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:33,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:33,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T15:17:33,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T15:17:33,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:33,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:33,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:33,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:33,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:33,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/4ac577cfb2f24d44b1dc237c4b9f63d3 2024-11-10T15:17:33,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/220447cefc8c44bdbeb6eafe17511ba7 is 50, key is test_row_0/C:col10/1731251851669/Put/seqid=0 2024-11-10T15:17:33,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742154_1330 (size=12301) 2024-11-10T15:17:33,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T15:17:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:33,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T15:17:33,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T15:17:33,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:33,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:33,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:33,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:33,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:33,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251913832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:33,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251913832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:33,836 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_2' in c42fcce8c0425e5889d52562aa8da5f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:33,835 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_2' in c42fcce8c0425e5889d52562aa8da5f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:33,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251913901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:33,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/220447cefc8c44bdbeb6eafe17511ba7 2024-11-10T15:17:33,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/83f3b33bb0804279b0d97c11db7e1aac as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/83f3b33bb0804279b0d97c11db7e1aac 2024-11-10T15:17:33,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/83f3b33bb0804279b0d97c11db7e1aac, entries=200, sequenceid=334, filesize=14.4 K 2024-11-10T15:17:33,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/4ac577cfb2f24d44b1dc237c4b9f63d3 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/4ac577cfb2f24d44b1dc237c4b9f63d3 2024-11-10T15:17:33,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/4ac577cfb2f24d44b1dc237c4b9f63d3, entries=150, sequenceid=334, filesize=12.0 K 2024-11-10T15:17:33,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/220447cefc8c44bdbeb6eafe17511ba7 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/220447cefc8c44bdbeb6eafe17511ba7 2024-11-10T15:17:33,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/220447cefc8c44bdbeb6eafe17511ba7, entries=150, sequenceid=334, filesize=12.0 K 2024-11-10T15:17:33,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c42fcce8c0425e5889d52562aa8da5f6 in 1277ms, sequenceid=334, compaction requested=false 2024-11-10T15:17:33,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:33,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T15:17:33,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:33,976 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T15:17:33,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:33,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:33,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:33,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:33,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:33,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:33,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/75b01a1fdf4941bc81e0b61f3729b334 is 50, key is test_row_0/A:col10/1731251852741/Put/seqid=0 2024-11-10T15:17:33,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742155_1331 (size=12301) 2024-11-10T15:17:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T15:17:34,385 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/75b01a1fdf4941bc81e0b61f3729b334 2024-11-10T15:17:34,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/2b197cfe5d4b4ba6b70b9fafde42bf53 is 50, key is test_row_0/B:col10/1731251852741/Put/seqid=0 2024-11-10T15:17:34,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742156_1332 (size=12301) 2024-11-10T15:17:34,396 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/2b197cfe5d4b4ba6b70b9fafde42bf53 2024-11-10T15:17:34,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/35114cda4a464c64bad01395165a4bfc is 50, key is test_row_0/C:col10/1731251852741/Put/seqid=0 2024-11-10T15:17:34,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742157_1333 (size=12301) 2024-11-10T15:17:34,407 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/35114cda4a464c64bad01395165a4bfc 2024-11-10T15:17:34,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/75b01a1fdf4941bc81e0b61f3729b334 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/75b01a1fdf4941bc81e0b61f3729b334 2024-11-10T15:17:34,417 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/75b01a1fdf4941bc81e0b61f3729b334, entries=150, sequenceid=352, filesize=12.0 K 2024-11-10T15:17:34,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/2b197cfe5d4b4ba6b70b9fafde42bf53 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/2b197cfe5d4b4ba6b70b9fafde42bf53 2024-11-10T15:17:34,423 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/2b197cfe5d4b4ba6b70b9fafde42bf53, entries=150, sequenceid=352, filesize=12.0 K 2024-11-10T15:17:34,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/35114cda4a464c64bad01395165a4bfc as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/35114cda4a464c64bad01395165a4bfc 2024-11-10T15:17:34,429 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/35114cda4a464c64bad01395165a4bfc, entries=150, sequenceid=352, filesize=12.0 K 2024-11-10T15:17:34,430 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for c42fcce8c0425e5889d52562aa8da5f6 in 453ms, sequenceid=352, compaction requested=true 2024-11-10T15:17:34,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:34,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:34,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-10T15:17:34,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-10T15:17:34,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-10T15:17:34,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0680 sec 2024-11-10T15:17:34,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0720 sec 2024-11-10T15:17:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T15:17:34,502 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:34,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:17:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-10T15:17:34,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T15:17:34,504 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:17:34,505 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:17:34,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:17:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T15:17:34,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T15:17:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-10T15:17:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-10T15:17:34,662 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-10T15:17:34,662 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-10T15:17:34,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 160 msec 2024-11-10T15:17:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T15:17:34,828 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:34,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:17:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-10T15:17:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T15:17:34,831 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:17:34,831 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:17:34,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:17:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T15:17:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:34,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T15:17:34,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:34,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:34,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:34,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:34,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:34,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:34,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/75058a6c27b649e79010b2a45607e3ae is 50, key is test_row_0/A:col10/1731251854943/Put/seqid=0 2024-11-10T15:17:34,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742158_1334 (size=14741) 2024-11-10T15:17:34,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T15:17:34,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:34,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:34,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:34,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:34,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251854971, max timestamp=1731251855048Num uniques 859; 2024-11-10T15:17:35,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:17:35,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251854971, max timestamp=1731251855048Num uniques 859; 2024-11-10T15:17:35,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:17:35,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251854971, max timestamp=1731251855048Num uniques 859; 2024-11-10T15:17:35,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:17:35,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T15:17:35,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:35,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:35,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:35,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T15:17:35,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251915164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:35,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251915272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:35,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T15:17:35,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:35,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:35,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:35,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/75058a6c27b649e79010b2a45607e3ae 2024-11-10T15:17:35,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/388d0ee0354949ccacfe6f3d45b38dd2 is 50, key is test_row_0/B:col10/1731251854943/Put/seqid=0 2024-11-10T15:17:35,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742159_1335 (size=12301) 2024-11-10T15:17:35,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T15:17:35,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:35,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:35,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:35,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T15:17:35,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251915481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:35,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:35,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:35,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:35,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T15:17:35,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:35,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:35,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:35,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T15:17:35,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:35,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:35,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:35,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/388d0ee0354949ccacfe6f3d45b38dd2 2024-11-10T15:17:35,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/200b2a38e05b4f3cb0baec8c053a8df4 is 50, key is test_row_0/C:col10/1731251854943/Put/seqid=0 2024-11-10T15:17:35,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742160_1336 (size=12301) 2024-11-10T15:17:35,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251915791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:35,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:35,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:35,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:35,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731251915841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:35,845 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_2' in c42fcce8c0425e5889d52562aa8da5f6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:35,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38618 deadline: 1731251915862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:35,866 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_2' in c42fcce8c0425e5889d52562aa8da5f6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:35,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T15:17:35,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:35,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:35,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:35,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T15:17:36,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T15:17:36,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:36,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:36,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:36,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:36,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:36,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/200b2a38e05b4f3cb0baec8c053a8df4 2024-11-10T15:17:36,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/75058a6c27b649e79010b2a45607e3ae as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/75058a6c27b649e79010b2a45607e3ae 2024-11-10T15:17:36,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/75058a6c27b649e79010b2a45607e3ae, entries=200, sequenceid=363, filesize=14.4 K 2024-11-10T15:17:36,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/388d0ee0354949ccacfe6f3d45b38dd2 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/388d0ee0354949ccacfe6f3d45b38dd2 2024-11-10T15:17:36,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/388d0ee0354949ccacfe6f3d45b38dd2, entries=150, sequenceid=363, filesize=12.0 K 2024-11-10T15:17:36,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/200b2a38e05b4f3cb0baec8c053a8df4 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/200b2a38e05b4f3cb0baec8c053a8df4 2024-11-10T15:17:36,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/200b2a38e05b4f3cb0baec8c053a8df4, entries=150, sequenceid=363, filesize=12.0 K 2024-11-10T15:17:36,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c42fcce8c0425e5889d52562aa8da5f6 in 1246ms, sequenceid=363, compaction requested=true 2024-11-10T15:17:36,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:36,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:17:36,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:36,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:17:36,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:36,190 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:17:36,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:17:36,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:36,190 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:17:36,192 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:17:36,192 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:17:36,192 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/B is initiating minor compaction (all files) 2024-11-10T15:17:36,192 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/A is initiating minor compaction (all files) 2024-11-10T15:17:36,192 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/A in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:36,192 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/B in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:36,192 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/42dbd47e5d2740c3947966751db8a9a5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/83f3b33bb0804279b0d97c11db7e1aac, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/75b01a1fdf4941bc81e0b61f3729b334, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/75058a6c27b649e79010b2a45607e3ae] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=53.5 K 2024-11-10T15:17:36,192 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/b96ee92d812444ca9fe828b994722be2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/4ac577cfb2f24d44b1dc237c4b9f63d3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/2b197cfe5d4b4ba6b70b9fafde42bf53, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/388d0ee0354949ccacfe6f3d45b38dd2] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=48.8 K 2024-11-10T15:17:36,192 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting b96ee92d812444ca9fe828b994722be2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731251850466 2024-11-10T15:17:36,192 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42dbd47e5d2740c3947966751db8a9a5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731251850466 2024-11-10T15:17:36,192 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ac577cfb2f24d44b1dc237c4b9f63d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731251851664 2024-11-10T15:17:36,193 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83f3b33bb0804279b0d97c11db7e1aac, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731251851661 2024-11-10T15:17:36,193 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b197cfe5d4b4ba6b70b9fafde42bf53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731251852720 2024-11-10T15:17:36,193 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75b01a1fdf4941bc81e0b61f3729b334, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731251852720 2024-11-10T15:17:36,193 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 388d0ee0354949ccacfe6f3d45b38dd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731251854917 2024-11-10T15:17:36,193 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75058a6c27b649e79010b2a45607e3ae, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731251854917 2024-11-10T15:17:36,202 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#B#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:36,202 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/858068955fa746b99a3ed7b0a0c9fe88 is 50, key is test_row_0/B:col10/1731251854943/Put/seqid=0 2024-11-10T15:17:36,206 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#A#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:36,207 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/673d87fb8d9e4e64a20dcc9b54313828 is 50, key is test_row_0/A:col10/1731251854943/Put/seqid=0 2024-11-10T15:17:36,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T15:17:36,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:36,213 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-10T15:17:36,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:36,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:36,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:36,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:36,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:36,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:17:36,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/929e35a8da1440ce8cb79e80a895cc0a is 50, key is test_row_0/A:col10/1731251855162/Put/seqid=0 2024-11-10T15:17:36,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742161_1337 (size=13153) 2024-11-10T15:17:36,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742162_1338 (size=13153) 2024-11-10T15:17:36,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742163_1339 (size=12301) 2024-11-10T15:17:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:36,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. as already flushing 2024-11-10T15:17:36,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251916340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:36,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:36,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:36,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:36,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251916451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:36,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:36,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:36,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:36,637 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/858068955fa746b99a3ed7b0a0c9fe88 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/858068955fa746b99a3ed7b0a0c9fe88 2024-11-10T15:17:36,642 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/673d87fb8d9e4e64a20dcc9b54313828 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/673d87fb8d9e4e64a20dcc9b54313828 2024-11-10T15:17:36,642 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/B of c42fcce8c0425e5889d52562aa8da5f6 into 858068955fa746b99a3ed7b0a0c9fe88(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:36,642 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:36,642 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/B, priority=12, startTime=1731251856190; duration=0sec 2024-11-10T15:17:36,642 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:36,642 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:B 2024-11-10T15:17:36,642 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:17:36,643 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:17:36,643 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/C is initiating minor compaction (all files) 2024-11-10T15:17:36,643 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/C in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:36,643 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/484f0835dbc24c75a7164c6bdf151c69, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/220447cefc8c44bdbeb6eafe17511ba7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/35114cda4a464c64bad01395165a4bfc, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/200b2a38e05b4f3cb0baec8c053a8df4] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=48.8 K 2024-11-10T15:17:36,646 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/929e35a8da1440ce8cb79e80a895cc0a 2024-11-10T15:17:36,646 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/A of c42fcce8c0425e5889d52562aa8da5f6 into 673d87fb8d9e4e64a20dcc9b54313828(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:36,646 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:36,646 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/A, priority=12, startTime=1731251856190; duration=0sec 2024-11-10T15:17:36,646 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:36,647 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:A 2024-11-10T15:17:36,647 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 484f0835dbc24c75a7164c6bdf151c69, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731251850466 2024-11-10T15:17:36,647 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 220447cefc8c44bdbeb6eafe17511ba7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731251851664 2024-11-10T15:17:36,647 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 35114cda4a464c64bad01395165a4bfc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731251852720 2024-11-10T15:17:36,647 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 200b2a38e05b4f3cb0baec8c053a8df4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731251854917 2024-11-10T15:17:36,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/f9e67c632e764878a35624270c3c3a4a is 50, key is test_row_0/B:col10/1731251855162/Put/seqid=0 2024-11-10T15:17:36,655 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#C#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:36,655 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/c09d906a6b8f4d6aa307a18c4031405b is 50, key is test_row_0/C:col10/1731251854943/Put/seqid=0 2024-11-10T15:17:36,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251916661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:36,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:36,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:36,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:36,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742165_1341 (size=12301) 2024-11-10T15:17:36,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742164_1340 (size=13153) 2024-11-10T15:17:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T15:17:37,152 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/f9e67c632e764878a35624270c3c3a4a 2024-11-10T15:17:37,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38594 deadline: 1731251917150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:37,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38552 deadline: 1731251917151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:37,157 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in c42fcce8c0425e5889d52562aa8da5f6 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18408 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:17:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:17:37,159 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/c09d906a6b8f4d6aa307a18c4031405b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/c09d906a6b8f4d6aa307a18c4031405b 2024-11-10T15:17:37,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/3422da12f8bf40cf85656801310ea430 is 50, key is test_row_0/C:col10/1731251855162/Put/seqid=0 2024-11-10T15:17:37,163 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/C of c42fcce8c0425e5889d52562aa8da5f6 into c09d906a6b8f4d6aa307a18c4031405b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:37,163 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:37,163 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/C, priority=12, startTime=1731251856190; duration=0sec 2024-11-10T15:17:37,163 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:37,164 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:C 2024-11-10T15:17:37,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742166_1342 (size=12301) 2024-11-10T15:17:37,435 INFO [Thread-1221 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1221. 2024-11-10T15:17:37,435 INFO [Thread-1217 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1217. 2024-11-10T15:17:37,435 DEBUG [Thread-1217 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:37,435 DEBUG [Thread-1221 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:37,435 DEBUG [Thread-1217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:37,435 DEBUG [Thread-1221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:37,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:37,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:37,436 DEBUG [Thread-1217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:37,436 DEBUG [Thread-1221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:37,438 INFO [Thread-1215 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1215. 2024-11-10T15:17:37,438 DEBUG [Thread-1215 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:37,438 DEBUG [Thread-1215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:37,438 DEBUG [Thread-1215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:37,438 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:37,440 INFO [Thread-1219 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1219. 2024-11-10T15:17:37,440 DEBUG [Thread-1219 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:37,440 DEBUG [Thread-1219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:37,440 DEBUG [Thread-1219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:37,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:37,441 INFO [Thread-1213 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1213. 2024-11-10T15:17:37,441 DEBUG [Thread-1213 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:37,441 DEBUG [Thread-1213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:37,441 DEBUG [Thread-1213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:37,441 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:37,567 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/3422da12f8bf40cf85656801310ea430 2024-11-10T15:17:37,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/929e35a8da1440ce8cb79e80a895cc0a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/929e35a8da1440ce8cb79e80a895cc0a 2024-11-10T15:17:37,574 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/929e35a8da1440ce8cb79e80a895cc0a, entries=150, sequenceid=400, filesize=12.0 K 2024-11-10T15:17:37,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/f9e67c632e764878a35624270c3c3a4a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/f9e67c632e764878a35624270c3c3a4a 2024-11-10T15:17:37,578 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/f9e67c632e764878a35624270c3c3a4a, entries=150, sequenceid=400, filesize=12.0 K 2024-11-10T15:17:37,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/3422da12f8bf40cf85656801310ea430 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/3422da12f8bf40cf85656801310ea430 2024-11-10T15:17:37,583 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/3422da12f8bf40cf85656801310ea430, entries=150, sequenceid=400, filesize=12.0 K 2024-11-10T15:17:37,583 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for c42fcce8c0425e5889d52562aa8da5f6 in 1370ms, sequenceid=400, compaction requested=false 2024-11-10T15:17:37,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:37,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:37,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-10T15:17:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-10T15:17:37,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-10T15:17:37,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7530 sec 2024-11-10T15:17:37,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.7580 sec 2024-11-10T15:17:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:37,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T15:17:37,664 INFO [Thread-1202 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1202. 2024-11-10T15:17:37,664 DEBUG [Thread-1202 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:37,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:37,664 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:37,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:37,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:37,664 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:37,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:37,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:37,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:37,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:37,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/0dc444c9968742edb854da5bccfa3673 is 50, key is test_row_0/A:col10/1731251856339/Put/seqid=0 2024-11-10T15:17:37,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742167_1343 (size=12301) 2024-11-10T15:17:38,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/0dc444c9968742edb854da5bccfa3673 2024-11-10T15:17:38,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/afdeeefe556145179666c73542a4c609 is 50, key is test_row_0/B:col10/1731251856339/Put/seqid=0 2024-11-10T15:17:38,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742168_1344 (size=12301) 2024-11-10T15:17:38,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/afdeeefe556145179666c73542a4c609 2024-11-10T15:17:38,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/191bc70de424470fba851ca558d942e2 is 50, key is test_row_0/C:col10/1731251856339/Put/seqid=0 2024-11-10T15:17:38,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742169_1345 (size=12301) 2024-11-10T15:17:38,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/191bc70de424470fba851ca558d942e2 2024-11-10T15:17:38,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/0dc444c9968742edb854da5bccfa3673 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/0dc444c9968742edb854da5bccfa3673 2024-11-10T15:17:38,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/0dc444c9968742edb854da5bccfa3673, entries=150, sequenceid=414, filesize=12.0 K 2024-11-10T15:17:38,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/afdeeefe556145179666c73542a4c609 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/afdeeefe556145179666c73542a4c609 2024-11-10T15:17:38,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/afdeeefe556145179666c73542a4c609, entries=150, sequenceid=414, filesize=12.0 K 2024-11-10T15:17:38,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/191bc70de424470fba851ca558d942e2 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/191bc70de424470fba851ca558d942e2 2024-11-10T15:17:38,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/191bc70de424470fba851ca558d942e2, entries=150, sequenceid=414, filesize=12.0 K 2024-11-10T15:17:38,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for c42fcce8c0425e5889d52562aa8da5f6 in 1271ms, sequenceid=414, compaction requested=true 2024-11-10T15:17:38,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:38,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:17:38,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:38,935 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:38,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:17:38,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:38,935 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:38,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c42fcce8c0425e5889d52562aa8da5f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:17:38,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:38,936 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:38,936 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:38,936 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/A is initiating minor compaction (all files) 2024-11-10T15:17:38,936 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/B is initiating minor compaction (all files) 2024-11-10T15:17:38,936 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/B in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:38,936 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/A in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:38,936 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/858068955fa746b99a3ed7b0a0c9fe88, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/f9e67c632e764878a35624270c3c3a4a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/afdeeefe556145179666c73542a4c609] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=36.9 K 2024-11-10T15:17:38,936 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/673d87fb8d9e4e64a20dcc9b54313828, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/929e35a8da1440ce8cb79e80a895cc0a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/0dc444c9968742edb854da5bccfa3673] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=36.9 K 2024-11-10T15:17:38,936 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 858068955fa746b99a3ed7b0a0c9fe88, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731251854917 2024-11-10T15:17:38,936 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 673d87fb8d9e4e64a20dcc9b54313828, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731251854917 2024-11-10T15:17:38,937 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 929e35a8da1440ce8cb79e80a895cc0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731251855082 2024-11-10T15:17:38,937 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting f9e67c632e764878a35624270c3c3a4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731251855082 2024-11-10T15:17:38,937 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dc444c9968742edb854da5bccfa3673, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731251856334 2024-11-10T15:17:38,937 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting afdeeefe556145179666c73542a4c609, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731251856334 2024-11-10T15:17:38,945 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#B#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:38,945 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#A#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:38,945 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/a9ea2760499a4db6b7f8a0a8c198472d is 50, key is test_row_0/B:col10/1731251856339/Put/seqid=0 2024-11-10T15:17:38,945 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/77b80d57c741446eb5b95ac8eb5b7eb9 is 50, key is test_row_0/A:col10/1731251856339/Put/seqid=0 2024-11-10T15:17:38,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742171_1347 (size=13255) 2024-11-10T15:17:38,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742170_1346 (size=13255) 2024-11-10T15:17:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T15:17:39,162 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:39,355 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/a9ea2760499a4db6b7f8a0a8c198472d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/a9ea2760499a4db6b7f8a0a8c198472d 2024-11-10T15:17:39,355 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/77b80d57c741446eb5b95ac8eb5b7eb9 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/77b80d57c741446eb5b95ac8eb5b7eb9 2024-11-10T15:17:39,359 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/B of c42fcce8c0425e5889d52562aa8da5f6 into a9ea2760499a4db6b7f8a0a8c198472d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:39,359 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/A of c42fcce8c0425e5889d52562aa8da5f6 into 77b80d57c741446eb5b95ac8eb5b7eb9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:39,359 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:39,359 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:39,359 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/B, priority=13, startTime=1731251858935; duration=0sec 2024-11-10T15:17:39,359 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/A, priority=13, startTime=1731251858935; duration=0sec 2024-11-10T15:17:39,359 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:39,359 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:B 2024-11-10T15:17:39,359 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:39,359 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:A 2024-11-10T15:17:39,359 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:39,360 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:39,360 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): c42fcce8c0425e5889d52562aa8da5f6/C is initiating minor compaction (all files) 2024-11-10T15:17:39,360 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c42fcce8c0425e5889d52562aa8da5f6/C in TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:39,360 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/c09d906a6b8f4d6aa307a18c4031405b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/3422da12f8bf40cf85656801310ea430, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/191bc70de424470fba851ca558d942e2] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp, totalSize=36.9 K 2024-11-10T15:17:39,360 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting c09d906a6b8f4d6aa307a18c4031405b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731251854917 2024-11-10T15:17:39,361 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3422da12f8bf40cf85656801310ea430, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731251855082 2024-11-10T15:17:39,361 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 191bc70de424470fba851ca558d942e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731251856334 2024-11-10T15:17:39,368 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c42fcce8c0425e5889d52562aa8da5f6#C#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:39,368 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/7ec2edf3164d48bba53e8674d31d045f is 50, key is test_row_0/C:col10/1731251856339/Put/seqid=0 2024-11-10T15:17:39,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742172_1348 (size=13255) 2024-11-10T15:17:39,425 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T15:17:39,777 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/7ec2edf3164d48bba53e8674d31d045f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/7ec2edf3164d48bba53e8674d31d045f 2024-11-10T15:17:39,784 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c42fcce8c0425e5889d52562aa8da5f6/C of c42fcce8c0425e5889d52562aa8da5f6 into 7ec2edf3164d48bba53e8674d31d045f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:39,784 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c42fcce8c0425e5889d52562aa8da5f6: 2024-11-10T15:17:39,784 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., storeName=c42fcce8c0425e5889d52562aa8da5f6/C, priority=13, startTime=1731251858935; duration=0sec 2024-11-10T15:17:39,784 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:39,784 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c42fcce8c0425e5889d52562aa8da5f6:C 2024-11-10T15:17:39,883 INFO [Thread-1204 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1204. 2024-11-10T15:17:39,883 DEBUG [Thread-1204 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:39,883 DEBUG [Thread-1204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:39,883 DEBUG [Thread-1204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:39,883 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:39,912 INFO [Thread-1208 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1208. 2024-11-10T15:17:39,912 DEBUG [Thread-1208 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:39,912 DEBUG [Thread-1208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:39,913 DEBUG [Thread-1208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:39,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:41,713 INFO [Thread-1210 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1210. 2024-11-10T15:17:41,713 DEBUG [Thread-1210 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:41,713 DEBUG [Thread-1210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:41,713 DEBUG [Thread-1210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:41,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:47,265 INFO [Thread-1206 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1206. 2024-11-10T15:17:47,265 DEBUG [Thread-1206 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:17:47,265 DEBUG [Thread-1206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:47,266 DEBUG [Thread-1206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:47,266 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:47,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T15:17:47,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 199 2024-11-10T15:17:47,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-10T15:17:47,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-10T15:17:47,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-10T15:17:47,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-10T15:17:47,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T15:17:47,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2759 2024-11-10T15:17:47,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2773 2024-11-10T15:17:47,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2891 2024-11-10T15:17:47,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2753 2024-11-10T15:17:47,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2776 2024-11-10T15:17:47,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T15:17:47,267 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T15:17:47,267 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T15:17:47,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:47,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:47,267 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:17:47,269 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T15:17:47,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T15:17:47,273 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251867273"}]},"ts":"1731251867273"} 2024-11-10T15:17:47,275 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T15:17:47,275 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T15:17:47,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T15:17:47,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c42fcce8c0425e5889d52562aa8da5f6, UNASSIGN}] 2024-11-10T15:17:47,279 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c42fcce8c0425e5889d52562aa8da5f6, UNASSIGN 2024-11-10T15:17:47,280 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=c42fcce8c0425e5889d52562aa8da5f6, regionState=CLOSING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:17:47,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c42fcce8c0425e5889d52562aa8da5f6, UNASSIGN because future has completed 2024-11-10T15:17:47,282 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T15:17:47,283 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:17:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T15:17:47,436 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:47,436 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T15:17:47,436 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing c42fcce8c0425e5889d52562aa8da5f6, disabling compactions & flushes 2024-11-10T15:17:47,436 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:47,436 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:47,436 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. after waiting 0 ms 2024-11-10T15:17:47,436 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:47,436 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing c42fcce8c0425e5889d52562aa8da5f6 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-10T15:17:47,436 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=A 2024-11-10T15:17:47,436 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:47,436 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=B 2024-11-10T15:17:47,437 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:47,437 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c42fcce8c0425e5889d52562aa8da5f6, store=C 2024-11-10T15:17:47,437 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:47,441 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/a7260734c724438e8f2c5cc2599ee0da is 50, key is test_row_0/A:col10/1731251867263/Put/seqid=0 2024-11-10T15:17:47,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742173_1349 (size=12301) 2024-11-10T15:17:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T15:17:47,846 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/a7260734c724438e8f2c5cc2599ee0da 2024-11-10T15:17:47,853 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/3f422aea2694455b8dc76ff103a184bc is 50, key is test_row_0/B:col10/1731251867263/Put/seqid=0 2024-11-10T15:17:47,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742174_1350 (size=12301) 2024-11-10T15:17:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T15:17:48,257 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/3f422aea2694455b8dc76ff103a184bc 2024-11-10T15:17:48,264 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/6eb6473baef34f93840111376b41a0bf is 50, key is test_row_0/C:col10/1731251867263/Put/seqid=0 2024-11-10T15:17:48,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742175_1351 (size=12301) 2024-11-10T15:17:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T15:17:48,668 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/6eb6473baef34f93840111376b41a0bf 2024-11-10T15:17:48,675 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/A/a7260734c724438e8f2c5cc2599ee0da as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/a7260734c724438e8f2c5cc2599ee0da 2024-11-10T15:17:48,682 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/a7260734c724438e8f2c5cc2599ee0da, entries=150, sequenceid=424, filesize=12.0 K 2024-11-10T15:17:48,682 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/B/3f422aea2694455b8dc76ff103a184bc as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/3f422aea2694455b8dc76ff103a184bc 2024-11-10T15:17:48,687 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/3f422aea2694455b8dc76ff103a184bc, entries=150, sequenceid=424, filesize=12.0 K 2024-11-10T15:17:48,688 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/.tmp/C/6eb6473baef34f93840111376b41a0bf as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/6eb6473baef34f93840111376b41a0bf 2024-11-10T15:17:48,692 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/6eb6473baef34f93840111376b41a0bf, entries=150, sequenceid=424, filesize=12.0 K 2024-11-10T15:17:48,693 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c42fcce8c0425e5889d52562aa8da5f6 in 1256ms, sequenceid=424, compaction requested=false 2024-11-10T15:17:48,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/1d1285dde25e4e27aa685fae993a4ff5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e42cf7a9e15b431d850cf1b95a4964ad, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/d556a01c60e945708404461b4119c313, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/c7ca8d5dcc8e4d4b9487c072aa5d4361, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/63301fcc19184cc5b4721cf02d92917c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/5cc037b209ef4000bd0d420857af71e4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/3223faa581644985b184878cdf967577, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/8544ee26a18047fe87b3e4c6ff429c16, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/08215547116940cba6202fea3bee1043, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/716519a7a0d64101a1e8a2ac386713f8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/c002291e73e44a3c96909e354bbbcb27, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/2b32d537367044f38f2a12cc1d8bab4d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/d66f6037326444d9a247bb4f9debde5b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e08017fbb0fb4430a6dcca7ab058a330, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e4ce55cf6b9141e4ab7a899ef8bb4033, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/f55d924f82dc43e18d194c64d6e52789, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/70918db59c78473aa6242195fe4892a5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/4b7ddda1bd3447bfb6bda5b23f74d6fb, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/7d2a3c71e9674bc89fb15417c133f026, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/4bad4de814d14d39a42e19dcd36df95d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/413e860c62c14b3bb358357c493220b7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/42dbd47e5d2740c3947966751db8a9a5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/83f3b33bb0804279b0d97c11db7e1aac, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/75b01a1fdf4941bc81e0b61f3729b334, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/75058a6c27b649e79010b2a45607e3ae, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/673d87fb8d9e4e64a20dcc9b54313828, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/929e35a8da1440ce8cb79e80a895cc0a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/0dc444c9968742edb854da5bccfa3673] to archive 2024-11-10T15:17:48,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:17:48,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/1d1285dde25e4e27aa685fae993a4ff5 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/1d1285dde25e4e27aa685fae993a4ff5 2024-11-10T15:17:48,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e42cf7a9e15b431d850cf1b95a4964ad to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e42cf7a9e15b431d850cf1b95a4964ad 2024-11-10T15:17:48,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/d556a01c60e945708404461b4119c313 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/d556a01c60e945708404461b4119c313 2024-11-10T15:17:48,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/c7ca8d5dcc8e4d4b9487c072aa5d4361 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/c7ca8d5dcc8e4d4b9487c072aa5d4361 2024-11-10T15:17:48,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/63301fcc19184cc5b4721cf02d92917c to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/63301fcc19184cc5b4721cf02d92917c 2024-11-10T15:17:48,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/5cc037b209ef4000bd0d420857af71e4 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/5cc037b209ef4000bd0d420857af71e4 2024-11-10T15:17:48,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/3223faa581644985b184878cdf967577 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/3223faa581644985b184878cdf967577 2024-11-10T15:17:48,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/8544ee26a18047fe87b3e4c6ff429c16 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/8544ee26a18047fe87b3e4c6ff429c16 2024-11-10T15:17:48,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/08215547116940cba6202fea3bee1043 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/08215547116940cba6202fea3bee1043 2024-11-10T15:17:48,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/716519a7a0d64101a1e8a2ac386713f8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/716519a7a0d64101a1e8a2ac386713f8 2024-11-10T15:17:48,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/c002291e73e44a3c96909e354bbbcb27 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/c002291e73e44a3c96909e354bbbcb27 2024-11-10T15:17:48,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/2b32d537367044f38f2a12cc1d8bab4d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/2b32d537367044f38f2a12cc1d8bab4d 2024-11-10T15:17:48,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/d66f6037326444d9a247bb4f9debde5b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/d66f6037326444d9a247bb4f9debde5b 2024-11-10T15:17:48,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e08017fbb0fb4430a6dcca7ab058a330 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e08017fbb0fb4430a6dcca7ab058a330 2024-11-10T15:17:48,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e4ce55cf6b9141e4ab7a899ef8bb4033 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/e4ce55cf6b9141e4ab7a899ef8bb4033 2024-11-10T15:17:48,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/f55d924f82dc43e18d194c64d6e52789 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/f55d924f82dc43e18d194c64d6e52789 2024-11-10T15:17:48,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/70918db59c78473aa6242195fe4892a5 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/70918db59c78473aa6242195fe4892a5 2024-11-10T15:17:48,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/4b7ddda1bd3447bfb6bda5b23f74d6fb to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/4b7ddda1bd3447bfb6bda5b23f74d6fb 2024-11-10T15:17:48,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/7d2a3c71e9674bc89fb15417c133f026 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/7d2a3c71e9674bc89fb15417c133f026 2024-11-10T15:17:48,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/4bad4de814d14d39a42e19dcd36df95d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/4bad4de814d14d39a42e19dcd36df95d 2024-11-10T15:17:48,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/413e860c62c14b3bb358357c493220b7 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/413e860c62c14b3bb358357c493220b7 2024-11-10T15:17:48,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/42dbd47e5d2740c3947966751db8a9a5 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/42dbd47e5d2740c3947966751db8a9a5 2024-11-10T15:17:48,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/83f3b33bb0804279b0d97c11db7e1aac to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/83f3b33bb0804279b0d97c11db7e1aac 2024-11-10T15:17:48,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/75b01a1fdf4941bc81e0b61f3729b334 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/75b01a1fdf4941bc81e0b61f3729b334 2024-11-10T15:17:48,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/75058a6c27b649e79010b2a45607e3ae to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/75058a6c27b649e79010b2a45607e3ae 2024-11-10T15:17:48,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/673d87fb8d9e4e64a20dcc9b54313828 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/673d87fb8d9e4e64a20dcc9b54313828 2024-11-10T15:17:48,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/929e35a8da1440ce8cb79e80a895cc0a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/929e35a8da1440ce8cb79e80a895cc0a 2024-11-10T15:17:48,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/0dc444c9968742edb854da5bccfa3673 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/0dc444c9968742edb854da5bccfa3673 2024-11-10T15:17:48,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/1ec579dc1f4642528f2bb85c53afa5ce, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/b0d725ce8a32454fa5fa895633e45948, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/c5470e2881e842eb9b32a218f88a3313, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/14037ffb6fb042e7a1871e3414200be0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/781dfa37e29745c4aeaa7cbf6b4b1060, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/47eea70f486443ef9b4e0e42d36d72b9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/d05a78002409473d9dda01fc8e54fde5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/64a87411b744422ea5a05e021fba754d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/35c547115c214d9bb4c45bb17a9c426b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/41f9bce7d70c49a0a12077390ab8cc4d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/966e185dedcc41df8e2a4327b28b0897, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/9b019df6c0854791afb50e8dcca5fe0b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/f81a9bc4f3b4474bb661971c16cacee2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/8362f557341a4b79932873d843f73962, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/1a79edea1c0e42989b052604760a4e68, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/7cc0a18d3da746189a9aa6937d88e582, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/18e2fedeb49049ef8338c4aaec681bb8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/e1f48aa8a2cf4500aee987b391e97d6e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/e95423041084484fa3f9be73af830f3a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/01369bb61fec45c1a934142f454fefc6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/b96ee92d812444ca9fe828b994722be2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/c9f18c1045ab4176b3cf0257d3cce212, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/4ac577cfb2f24d44b1dc237c4b9f63d3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/2b197cfe5d4b4ba6b70b9fafde42bf53, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/858068955fa746b99a3ed7b0a0c9fe88, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/388d0ee0354949ccacfe6f3d45b38dd2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/f9e67c632e764878a35624270c3c3a4a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/afdeeefe556145179666c73542a4c609] to archive 2024-11-10T15:17:48,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:17:48,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/1ec579dc1f4642528f2bb85c53afa5ce to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/1ec579dc1f4642528f2bb85c53afa5ce 2024-11-10T15:17:48,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/b0d725ce8a32454fa5fa895633e45948 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/b0d725ce8a32454fa5fa895633e45948 2024-11-10T15:17:48,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/c5470e2881e842eb9b32a218f88a3313 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/c5470e2881e842eb9b32a218f88a3313 2024-11-10T15:17:48,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/14037ffb6fb042e7a1871e3414200be0 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/14037ffb6fb042e7a1871e3414200be0 2024-11-10T15:17:48,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/781dfa37e29745c4aeaa7cbf6b4b1060 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/781dfa37e29745c4aeaa7cbf6b4b1060 2024-11-10T15:17:48,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/47eea70f486443ef9b4e0e42d36d72b9 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/47eea70f486443ef9b4e0e42d36d72b9 2024-11-10T15:17:48,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/d05a78002409473d9dda01fc8e54fde5 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/d05a78002409473d9dda01fc8e54fde5 2024-11-10T15:17:48,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/64a87411b744422ea5a05e021fba754d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/64a87411b744422ea5a05e021fba754d 2024-11-10T15:17:48,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/35c547115c214d9bb4c45bb17a9c426b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/35c547115c214d9bb4c45bb17a9c426b 2024-11-10T15:17:48,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/41f9bce7d70c49a0a12077390ab8cc4d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/41f9bce7d70c49a0a12077390ab8cc4d 2024-11-10T15:17:48,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/966e185dedcc41df8e2a4327b28b0897 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/966e185dedcc41df8e2a4327b28b0897 2024-11-10T15:17:48,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/9b019df6c0854791afb50e8dcca5fe0b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/9b019df6c0854791afb50e8dcca5fe0b 2024-11-10T15:17:48,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/f81a9bc4f3b4474bb661971c16cacee2 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/f81a9bc4f3b4474bb661971c16cacee2 2024-11-10T15:17:48,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/8362f557341a4b79932873d843f73962 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/8362f557341a4b79932873d843f73962 2024-11-10T15:17:48,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/1a79edea1c0e42989b052604760a4e68 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/1a79edea1c0e42989b052604760a4e68 2024-11-10T15:17:48,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/7cc0a18d3da746189a9aa6937d88e582 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/7cc0a18d3da746189a9aa6937d88e582 2024-11-10T15:17:48,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/18e2fedeb49049ef8338c4aaec681bb8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/18e2fedeb49049ef8338c4aaec681bb8 2024-11-10T15:17:48,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/e1f48aa8a2cf4500aee987b391e97d6e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/e1f48aa8a2cf4500aee987b391e97d6e 2024-11-10T15:17:48,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/e95423041084484fa3f9be73af830f3a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/e95423041084484fa3f9be73af830f3a 2024-11-10T15:17:48,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/01369bb61fec45c1a934142f454fefc6 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/01369bb61fec45c1a934142f454fefc6 2024-11-10T15:17:48,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/b96ee92d812444ca9fe828b994722be2 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/b96ee92d812444ca9fe828b994722be2 2024-11-10T15:17:48,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/c9f18c1045ab4176b3cf0257d3cce212 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/c9f18c1045ab4176b3cf0257d3cce212 2024-11-10T15:17:48,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/4ac577cfb2f24d44b1dc237c4b9f63d3 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/4ac577cfb2f24d44b1dc237c4b9f63d3 2024-11-10T15:17:48,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/2b197cfe5d4b4ba6b70b9fafde42bf53 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/2b197cfe5d4b4ba6b70b9fafde42bf53 2024-11-10T15:17:48,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/858068955fa746b99a3ed7b0a0c9fe88 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/858068955fa746b99a3ed7b0a0c9fe88 2024-11-10T15:17:48,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/388d0ee0354949ccacfe6f3d45b38dd2 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/388d0ee0354949ccacfe6f3d45b38dd2 2024-11-10T15:17:48,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/f9e67c632e764878a35624270c3c3a4a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/f9e67c632e764878a35624270c3c3a4a 2024-11-10T15:17:48,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/afdeeefe556145179666c73542a4c609 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/afdeeefe556145179666c73542a4c609 2024-11-10T15:17:48,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/14fb2f6a71a841f9ae109e4c026c6c66, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/fce1c3814c14411f8954717fa70a6ba8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/0df113a31c3742048a94b7fcba592f6a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/b9f360ea5e6a400ba0d76f9df90cc73a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/39eee88dacc64bcba2f2b94e8e9a5f26, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/5ac3ca9040c44b9682b17e2b10431a7c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/987c9a1b530a4a6c8ebbeaf1244589fa, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/a17c5381927f46f18db8fe725318fdd4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/b9da804cd8ab4bfda9ee92f9a3447091, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/e79f9b158b3a4fc48959d4773af39a29, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/f80b7b159c1a4af5a43919f50dccd865, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/04d4a832e1c14fb0aa827b157e13038a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/09a8ab3100f042a7a067f30d1480b552, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/bf2471d1f2a94e3596f13023c1c1c63f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/d42417e3a7c14f0f9593f5011b7ccda4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/210620fd1e1e447c8dbb078142d3a5cb, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/4010911c41cd49878afe16fff097faf8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/04ed0ea2fab64f3f96c163775d56928a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/6d0cc7f24de945618f961404497ce924, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/0451e88d49f741c4a3e0aebee36a8581, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/484f0835dbc24c75a7164c6bdf151c69, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/44267dda381245c195b217e08b810111, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/220447cefc8c44bdbeb6eafe17511ba7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/35114cda4a464c64bad01395165a4bfc, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/c09d906a6b8f4d6aa307a18c4031405b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/200b2a38e05b4f3cb0baec8c053a8df4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/3422da12f8bf40cf85656801310ea430, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/191bc70de424470fba851ca558d942e2] to archive 2024-11-10T15:17:48,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:17:48,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/14fb2f6a71a841f9ae109e4c026c6c66 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/14fb2f6a71a841f9ae109e4c026c6c66 2024-11-10T15:17:48,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/fce1c3814c14411f8954717fa70a6ba8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/fce1c3814c14411f8954717fa70a6ba8 2024-11-10T15:17:48,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/0df113a31c3742048a94b7fcba592f6a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/0df113a31c3742048a94b7fcba592f6a 2024-11-10T15:17:48,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/b9f360ea5e6a400ba0d76f9df90cc73a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/b9f360ea5e6a400ba0d76f9df90cc73a 2024-11-10T15:17:48,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/39eee88dacc64bcba2f2b94e8e9a5f26 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/39eee88dacc64bcba2f2b94e8e9a5f26 2024-11-10T15:17:48,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/5ac3ca9040c44b9682b17e2b10431a7c to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/5ac3ca9040c44b9682b17e2b10431a7c 2024-11-10T15:17:48,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/987c9a1b530a4a6c8ebbeaf1244589fa to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/987c9a1b530a4a6c8ebbeaf1244589fa 2024-11-10T15:17:48,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/a17c5381927f46f18db8fe725318fdd4 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/a17c5381927f46f18db8fe725318fdd4 2024-11-10T15:17:48,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/b9da804cd8ab4bfda9ee92f9a3447091 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/b9da804cd8ab4bfda9ee92f9a3447091 2024-11-10T15:17:48,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/e79f9b158b3a4fc48959d4773af39a29 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/e79f9b158b3a4fc48959d4773af39a29 2024-11-10T15:17:48,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/f80b7b159c1a4af5a43919f50dccd865 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/f80b7b159c1a4af5a43919f50dccd865 2024-11-10T15:17:48,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/04d4a832e1c14fb0aa827b157e13038a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/04d4a832e1c14fb0aa827b157e13038a 2024-11-10T15:17:48,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/09a8ab3100f042a7a067f30d1480b552 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/09a8ab3100f042a7a067f30d1480b552 2024-11-10T15:17:48,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/bf2471d1f2a94e3596f13023c1c1c63f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/bf2471d1f2a94e3596f13023c1c1c63f 2024-11-10T15:17:48,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/d42417e3a7c14f0f9593f5011b7ccda4 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/d42417e3a7c14f0f9593f5011b7ccda4 2024-11-10T15:17:48,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/210620fd1e1e447c8dbb078142d3a5cb to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/210620fd1e1e447c8dbb078142d3a5cb 2024-11-10T15:17:48,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/4010911c41cd49878afe16fff097faf8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/4010911c41cd49878afe16fff097faf8 2024-11-10T15:17:48,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/04ed0ea2fab64f3f96c163775d56928a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/04ed0ea2fab64f3f96c163775d56928a 2024-11-10T15:17:48,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/6d0cc7f24de945618f961404497ce924 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/6d0cc7f24de945618f961404497ce924 2024-11-10T15:17:48,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/0451e88d49f741c4a3e0aebee36a8581 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/0451e88d49f741c4a3e0aebee36a8581 2024-11-10T15:17:48,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/484f0835dbc24c75a7164c6bdf151c69 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/484f0835dbc24c75a7164c6bdf151c69 2024-11-10T15:17:48,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/44267dda381245c195b217e08b810111 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/44267dda381245c195b217e08b810111 2024-11-10T15:17:48,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/220447cefc8c44bdbeb6eafe17511ba7 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/220447cefc8c44bdbeb6eafe17511ba7 2024-11-10T15:17:48,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/35114cda4a464c64bad01395165a4bfc to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/35114cda4a464c64bad01395165a4bfc 2024-11-10T15:17:48,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/c09d906a6b8f4d6aa307a18c4031405b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/c09d906a6b8f4d6aa307a18c4031405b 2024-11-10T15:17:48,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/200b2a38e05b4f3cb0baec8c053a8df4 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/200b2a38e05b4f3cb0baec8c053a8df4 2024-11-10T15:17:48,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/3422da12f8bf40cf85656801310ea430 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/3422da12f8bf40cf85656801310ea430 2024-11-10T15:17:48,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/191bc70de424470fba851ca558d942e2 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/191bc70de424470fba851ca558d942e2 2024-11-10T15:17:48,776 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/recovered.edits/427.seqid, newMaxSeqId=427, maxSeqId=1 2024-11-10T15:17:48,777 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. 2024-11-10T15:17:48,777 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for c42fcce8c0425e5889d52562aa8da5f6: Waiting for close lock at 1731251867436Running coprocessor pre-close hooks at 1731251867436Disabling compacts and flushes for region at 1731251867436Disabling writes for close at 1731251867436Obtaining lock to block concurrent updates at 1731251867436Preparing flush snapshotting stores in c42fcce8c0425e5889d52562aa8da5f6 at 1731251867436Finished memstore snapshotting TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731251867437 (+1 ms)Flushing stores of TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6. at 1731251867437Flushing c42fcce8c0425e5889d52562aa8da5f6/A: creating writer at 1731251867437Flushing c42fcce8c0425e5889d52562aa8da5f6/A: appending metadata at 1731251867440 (+3 ms)Flushing c42fcce8c0425e5889d52562aa8da5f6/A: closing flushed file at 1731251867440Flushing c42fcce8c0425e5889d52562aa8da5f6/B: creating writer at 1731251867850 (+410 ms)Flushing c42fcce8c0425e5889d52562aa8da5f6/B: appending metadata at 1731251867853 (+3 ms)Flushing c42fcce8c0425e5889d52562aa8da5f6/B: closing flushed file at 1731251867853Flushing c42fcce8c0425e5889d52562aa8da5f6/C: creating writer at 1731251868261 (+408 ms)Flushing c42fcce8c0425e5889d52562aa8da5f6/C: appending metadata at 1731251868264 (+3 ms)Flushing c42fcce8c0425e5889d52562aa8da5f6/C: closing flushed file at 1731251868264Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4540b367: reopening flushed file at 1731251868674 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41668ad4: reopening flushed file at 1731251868682 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@369c0e70: reopening flushed file at 1731251868687 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c42fcce8c0425e5889d52562aa8da5f6 in 1256ms, sequenceid=424, compaction requested=false at 1731251868693 (+6 ms)Writing region close event to WAL at 1731251868774 (+81 ms)Running coprocessor post-close hooks at 1731251868777 (+3 ms)Closed at 1731251868777 2024-11-10T15:17:48,778 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:48,779 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=c42fcce8c0425e5889d52562aa8da5f6, regionState=CLOSED 2024-11-10T15:17:48,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:17:48,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-10T15:17:48,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure c42fcce8c0425e5889d52562aa8da5f6, server=c0771061be61,46087,1731251771284 in 1.4990 sec 2024-11-10T15:17:48,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-10T15:17:48,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c42fcce8c0425e5889d52562aa8da5f6, UNASSIGN in 1.5050 sec 2024-11-10T15:17:48,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-10T15:17:48,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5090 sec 2024-11-10T15:17:48,787 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251868787"}]},"ts":"1731251868787"} 2024-11-10T15:17:48,789 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T15:17:48,789 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T15:17:48,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5200 sec 2024-11-10T15:17:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T15:17:49,421 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:49,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T15:17:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:49,424 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T15:17:49,424 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:49,426 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:49,428 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/recovered.edits] 2024-11-10T15:17:49,430 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/77b80d57c741446eb5b95ac8eb5b7eb9 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/77b80d57c741446eb5b95ac8eb5b7eb9 2024-11-10T15:17:49,431 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/a7260734c724438e8f2c5cc2599ee0da to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/A/a7260734c724438e8f2c5cc2599ee0da 2024-11-10T15:17:49,433 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/3f422aea2694455b8dc76ff103a184bc to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/3f422aea2694455b8dc76ff103a184bc 2024-11-10T15:17:49,434 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/a9ea2760499a4db6b7f8a0a8c198472d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/B/a9ea2760499a4db6b7f8a0a8c198472d 2024-11-10T15:17:49,436 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/6eb6473baef34f93840111376b41a0bf to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/6eb6473baef34f93840111376b41a0bf 2024-11-10T15:17:49,437 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/7ec2edf3164d48bba53e8674d31d045f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/C/7ec2edf3164d48bba53e8674d31d045f 2024-11-10T15:17:49,439 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/recovered.edits/427.seqid to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6/recovered.edits/427.seqid 2024-11-10T15:17:49,439 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/c42fcce8c0425e5889d52562aa8da5f6 2024-11-10T15:17:49,439 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T15:17:49,441 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:49,444 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T15:17:49,446 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T15:17:49,447 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:49,447 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T15:17:49,448 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731251869447"}]},"ts":"9223372036854775807"} 2024-11-10T15:17:49,450 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T15:17:49,450 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c42fcce8c0425e5889d52562aa8da5f6, NAME => 'TestAcidGuarantees,,1731251836235.c42fcce8c0425e5889d52562aa8da5f6.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T15:17:49,450 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T15:17:49,450 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731251869450"}]},"ts":"9223372036854775807"} 2024-11-10T15:17:49,452 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T15:17:49,452 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:49,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-10T15:17:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T15:17:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T15:17:49,532 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:49,543 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=449 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=678 (was 808), ProcessCount=11 (was 11), AvailableMemoryMB=3730 (was 2850) - AvailableMemoryMB LEAK? - 2024-11-10T15:17:49,556 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=678, ProcessCount=11, AvailableMemoryMB=3729 2024-11-10T15:17:49,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T15:17:49,558 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T15:17:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:49,561 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T15:17:49,561 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:49,561 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-10T15:17:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T15:17:49,562 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T15:17:49,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742176_1352 (size=1061) 2024-11-10T15:17:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T15:17:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T15:17:49,971 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85 2024-11-10T15:17:49,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742177_1353 (size=53) 2024-11-10T15:17:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T15:17:50,377 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:17:50,377 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b423d13644c67f79070bddaad3c47156, disabling compactions & flushes 2024-11-10T15:17:50,378 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:50,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:50,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. after waiting 0 ms 2024-11-10T15:17:50,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:50,378 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:50,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b423d13644c67f79070bddaad3c47156: Waiting for close lock at 1731251870377Disabling compacts and flushes for region at 1731251870377Disabling writes for close at 1731251870378 (+1 ms)Writing region close event to WAL at 1731251870378Closed at 1731251870378 2024-11-10T15:17:50,379 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T15:17:50,379 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731251870379"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731251870379"}]},"ts":"1731251870379"} 2024-11-10T15:17:50,382 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T15:17:50,383 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T15:17:50,383 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251870383"}]},"ts":"1731251870383"} 2024-11-10T15:17:50,385 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T15:17:50,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b423d13644c67f79070bddaad3c47156, ASSIGN}] 2024-11-10T15:17:50,386 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b423d13644c67f79070bddaad3c47156, ASSIGN 2024-11-10T15:17:50,387 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b423d13644c67f79070bddaad3c47156, ASSIGN; state=OFFLINE, location=c0771061be61,46087,1731251771284; forceNewPlan=false, retain=false 2024-11-10T15:17:50,538 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=b423d13644c67f79070bddaad3c47156, regionState=OPENING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:17:50,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b423d13644c67f79070bddaad3c47156, ASSIGN because future has completed 2024-11-10T15:17:50,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:17:50,696 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:50,697 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} 2024-11-10T15:17:50,697 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:50,697 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:17:50,697 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:50,697 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:50,699 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:50,700 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:17:50,700 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b423d13644c67f79070bddaad3c47156 columnFamilyName A 2024-11-10T15:17:50,700 DEBUG [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:50,701 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.HStore(327): Store=b423d13644c67f79070bddaad3c47156/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:17:50,701 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T15:17:50,703 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:17:50,703 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b423d13644c67f79070bddaad3c47156 columnFamilyName B 2024-11-10T15:17:50,703 DEBUG [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:50,709 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.HStore(327): Store=b423d13644c67f79070bddaad3c47156/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:17:50,709 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:50,711 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:17:50,711 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b423d13644c67f79070bddaad3c47156 columnFamilyName C 2024-11-10T15:17:50,711 DEBUG [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:50,712 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.HStore(327): Store=b423d13644c67f79070bddaad3c47156/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:17:50,712 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:50,712 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:50,713 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:50,713 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:50,715 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:50,715 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:50,715 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T15:17:50,717 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:50,719 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T15:17:50,720 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened b423d13644c67f79070bddaad3c47156; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65904985, jitterRate=-0.017939195036888123}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T15:17:50,720 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:50,720 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for b423d13644c67f79070bddaad3c47156: Running coprocessor pre-open hook at 1731251870698Writing region info on filesystem at 1731251870698Initializing all the Stores at 1731251870698Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251870698Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251870698Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251870699 (+1 ms)Cleaning up temporary data from old regions at 1731251870715 (+16 ms)Running coprocessor post-open hooks at 1731251870720 (+5 ms)Region opened successfully at 1731251870720 2024-11-10T15:17:50,722 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., pid=91, masterSystemTime=1731251870693 2024-11-10T15:17:50,725 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:50,725 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:50,725 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=b423d13644c67f79070bddaad3c47156, regionState=OPEN, openSeqNum=2, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:17:50,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:17:50,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-10T15:17:50,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 in 188 msec 2024-11-10T15:17:50,736 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T15:17:50,737 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251870736"}]},"ts":"1731251870736"} 2024-11-10T15:17:50,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-10T15:17:50,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b423d13644c67f79070bddaad3c47156, ASSIGN in 347 msec 2024-11-10T15:17:50,739 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T15:17:50,740 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T15:17:50,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-11-10T15:17:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T15:17:51,713 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:51,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:51,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:51,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:51,716 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:51,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:51,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:51,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d66eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:51,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:51,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:51,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:51,719 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bd49c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:51,721 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:51,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:51,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:51,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:51,725 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T15:17:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c0771061be61,41501,1731251770419 2024-11-10T15:17:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@762b0008 2024-11-10T15:17:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T15:17:51,728 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T15:17:51,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T15:17:51,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T15:17:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T15:17:51,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742178_1354 (size=1097) 2024-11-10T15:17:51,774 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T15:17:51,774 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T15:17:51,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T15:17:51,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b423d13644c67f79070bddaad3c47156, REOPEN/MOVE}] 2024-11-10T15:17:51,785 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b423d13644c67f79070bddaad3c47156, REOPEN/MOVE 2024-11-10T15:17:51,785 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=b423d13644c67f79070bddaad3c47156, regionState=CLOSING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:17:51,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b423d13644c67f79070bddaad3c47156, REOPEN/MOVE because future has completed 2024-11-10T15:17:51,789 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41501 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=c0771061be61,46087,1731251771284, table=TestAcidGuarantees, region=b423d13644c67f79070bddaad3c47156. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-10T15:17:51,792 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T15:17:51,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:17:51,953 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:51,953 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T15:17:51,953 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing b423d13644c67f79070bddaad3c47156, disabling compactions & flushes 2024-11-10T15:17:51,953 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:51,953 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:51,953 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. after waiting 0 ms 2024-11-10T15:17:51,953 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:52,004 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T15:17:52,010 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:52,010 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for b423d13644c67f79070bddaad3c47156: Waiting for close lock at 1731251871953Running coprocessor pre-close hooks at 1731251871953Disabling compacts and flushes for region at 1731251871953Disabling writes for close at 1731251871953Writing region close event to WAL at 1731251871960 (+7 ms)Running coprocessor post-close hooks at 1731251872009 (+49 ms)Closed at 1731251872010 (+1 ms) 2024-11-10T15:17:52,010 WARN [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: b423d13644c67f79070bddaad3c47156 to self. 2024-11-10T15:17:52,013 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=b423d13644c67f79070bddaad3c47156, regionState=CLOSED 2024-11-10T15:17:52,013 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:52,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:17:52,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-10T15:17:52,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 in 230 msec 2024-11-10T15:17:52,029 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b423d13644c67f79070bddaad3c47156, REOPEN/MOVE; state=CLOSED, location=c0771061be61,46087,1731251771284; forceNewPlan=false, retain=true 2024-11-10T15:17:52,185 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=b423d13644c67f79070bddaad3c47156, regionState=OPENING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:17:52,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b423d13644c67f79070bddaad3c47156, REOPEN/MOVE because future has completed 2024-11-10T15:17:52,192 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:17:52,367 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:52,367 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} 2024-11-10T15:17:52,367 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:52,367 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:17:52,367 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:52,368 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:52,376 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:52,385 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:17:52,385 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b423d13644c67f79070bddaad3c47156 columnFamilyName A 2024-11-10T15:17:52,396 DEBUG [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:52,398 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.HStore(327): Store=b423d13644c67f79070bddaad3c47156/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:17:52,398 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:52,399 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:17:52,400 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b423d13644c67f79070bddaad3c47156 columnFamilyName B 2024-11-10T15:17:52,400 DEBUG [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:52,400 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.HStore(327): Store=b423d13644c67f79070bddaad3c47156/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:17:52,400 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:52,401 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:17:52,401 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b423d13644c67f79070bddaad3c47156 columnFamilyName C 2024-11-10T15:17:52,401 DEBUG [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:52,404 INFO [StoreOpener-b423d13644c67f79070bddaad3c47156-1 {}] regionserver.HStore(327): Store=b423d13644c67f79070bddaad3c47156/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:17:52,405 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:52,405 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:52,406 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:52,407 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:52,412 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:52,412 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:52,413 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T15:17:52,417 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:52,418 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened b423d13644c67f79070bddaad3c47156; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67257075, jitterRate=0.002208516001701355}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T15:17:52,418 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:52,419 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for b423d13644c67f79070bddaad3c47156: Running coprocessor pre-open hook at 1731251872368Writing region info on filesystem at 1731251872368Initializing all the Stores at 1731251872369 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251872369Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251872372 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251872372Cleaning up temporary data from old regions at 1731251872412 (+40 ms)Running coprocessor post-open hooks at 1731251872418 (+6 ms)Region opened successfully at 1731251872419 (+1 ms) 2024-11-10T15:17:52,422 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., pid=96, masterSystemTime=1731251872356 2024-11-10T15:17:52,428 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:52,428 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:52,433 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=b423d13644c67f79070bddaad3c47156, regionState=OPEN, openSeqNum=5, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:17:52,440 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:17:52,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-10T15:17:52,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 in 254 msec 2024-11-10T15:17:52,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-10T15:17:52,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b423d13644c67f79070bddaad3c47156, REOPEN/MOVE in 668 msec 2024-11-10T15:17:52,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-10T15:17:52,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 684 msec 2024-11-10T15:17:52,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 748 msec 2024-11-10T15:17:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-10T15:17:52,493 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:52,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,515 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:52,515 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:52,517 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@342af2ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:52,520 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ba2051, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:52,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:52,549 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a0bb391, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:52,551 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cda3a51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,553 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:52,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:52,554 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:52,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:52,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a3a1514, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:52,558 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7831d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,565 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:52,565 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:52,569 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:52,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2630e663, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:52,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:52,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:52,571 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:52,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d130afa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:52,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:52,580 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@538adaca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:52,588 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47b326c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:52,593 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:52,599 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:52,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c16319f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:52,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:52,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:52,601 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@406f0c8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:52,605 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:52,606 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1751e615, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:52,608 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d331f4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,612 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:52,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:52,614 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@616544f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:52,617 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf2193d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:52,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:52,622 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eb7df86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:52,624 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59cff0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:17:52,629 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:17:52,630 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:17:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:17:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:17:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27b4dc1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:17:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:17:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:17:52,633 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:17:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1518405b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:17:52,638 DEBUG [Thread-1608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:52,639 DEBUG [Thread-1612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:52,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:52,643 DEBUG [Thread-1621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:52,644 DEBUG [Thread-1627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:52,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:17:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:52,649 DEBUG [Thread-1625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:52,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:52,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:52,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:52,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:52,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:52,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:52,655 DEBUG [Thread-1616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:52,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:17:52,656 DEBUG [Thread-1616 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:52,657 DEBUG [Thread-1616 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:52,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:17:52,659 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:17:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:17:52,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:17:52,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:17:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-10T15:17:52,669 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:17:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T15:17:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:52,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T15:17:52,671 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:17:52,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:17:52,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:17:52,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:52,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:17:52,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:52,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:17:52,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:52,683 DEBUG [Thread-1619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:52,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:52,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:52,686 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:17:52,691 DEBUG [Thread-1623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:52,692 DEBUG [Thread-1623 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:52,692 DEBUG [Thread-1623 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:52,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:52,697 DEBUG [Thread-1623 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:17:52,720 DEBUG [Thread-1610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:52,725 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:17:52,728 DEBUG [Thread-1614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:17:52,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:17:52,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:17:52,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:17:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:17:52,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251932752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:52,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103db4f0bb9e6041db833af8bf8a8f67f5_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251872671/Put/seqid=0 2024-11-10T15:17:52,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251932756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:52,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251932760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T15:17:52,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251932761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:52,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251932764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:52,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:52,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742179_1355 (size=9714) 2024-11-10T15:17:52,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T15:17:52,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:52,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:17:52,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:52,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:52,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:52,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251932871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:52,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251932882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:52,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251932883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:52,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251932883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:52,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251932884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T15:17:52,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T15:17:52,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:52,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:17:52,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:52,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:52,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251933095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:53,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251933105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:53,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251933105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:53,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251933106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251933106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:53,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:53,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T15:17:53,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:53,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:17:53,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:53,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,218 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:53,238 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103db4f0bb9e6041db833af8bf8a8f67f5_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103db4f0bb9e6041db833af8bf8a8f67f5_b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:53,248 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/a7b9ddb34c674c50bb8a8794e3ffbd13, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:17:53,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/a7b9ddb34c674c50bb8a8794e3ffbd13 is 175, key is test_row_0/A:col10/1731251872671/Put/seqid=0 2024-11-10T15:17:53,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742180_1356 (size=22361) 2024-11-10T15:17:53,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T15:17:53,293 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/a7b9ddb34c674c50bb8a8794e3ffbd13 2024-11-10T15:17:53,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T15:17:53,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:53,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:17:53,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:53,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/5a38518941ee48fda2bbd2ad29c25ccc is 50, key is test_row_0/B:col10/1731251872671/Put/seqid=0 2024-11-10T15:17:53,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742181_1357 (size=9657) 2024-11-10T15:17:53,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:53,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251933412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:53,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:53,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251933421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:53,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:53,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251933424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:53,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:53,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251933425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:53,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251933425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:53,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:53,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T15:17:53,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:53,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:17:53,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:53,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T15:17:53,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:53,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:17:53,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:53,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/5a38518941ee48fda2bbd2ad29c25ccc 2024-11-10T15:17:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T15:17:53,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T15:17:53,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:53,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:17:53,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:53,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/9428dd32d4c6438b886a297eb4615e79 is 50, key is test_row_0/C:col10/1731251872671/Put/seqid=0 2024-11-10T15:17:53,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742182_1358 (size=9657) 2024-11-10T15:17:53,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251933921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:53,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251933942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:53,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:53,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251933942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:53,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251933944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:53,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251933944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:53,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T15:17:53,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:53,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:17:53,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:53,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:54,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T15:17:54,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:54,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:17:54,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:54,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:54,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:54,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T15:17:54,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:54,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:17:54,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:54,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:54,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:54,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/9428dd32d4c6438b886a297eb4615e79 2024-11-10T15:17:54,337 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T15:17:54,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/a7b9ddb34c674c50bb8a8794e3ffbd13 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/a7b9ddb34c674c50bb8a8794e3ffbd13 2024-11-10T15:17:54,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/a7b9ddb34c674c50bb8a8794e3ffbd13, entries=100, sequenceid=16, filesize=21.8 K 2024-11-10T15:17:54,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/5a38518941ee48fda2bbd2ad29c25ccc as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/5a38518941ee48fda2bbd2ad29c25ccc 2024-11-10T15:17:54,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/5a38518941ee48fda2bbd2ad29c25ccc, entries=100, sequenceid=16, filesize=9.4 K 2024-11-10T15:17:54,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/9428dd32d4c6438b886a297eb4615e79 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/9428dd32d4c6438b886a297eb4615e79 2024-11-10T15:17:54,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/9428dd32d4c6438b886a297eb4615e79, entries=100, sequenceid=16, filesize=9.4 K 2024-11-10T15:17:54,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b423d13644c67f79070bddaad3c47156 in 1731ms, sequenceid=16, compaction requested=false 2024-11-10T15:17:54,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:17:54,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T15:17:54,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:54,449 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T15:17:54,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:17:54,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:54,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:17:54,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:54,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:17:54,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:54,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110742c52f005534ae687806e3dfa4ab900_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251872757/Put/seqid=0 2024-11-10T15:17:54,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742183_1359 (size=12154) 2024-11-10T15:17:54,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:54,519 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110742c52f005534ae687806e3dfa4ab900_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110742c52f005534ae687806e3dfa4ab900_b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:54,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/e284de4042414398a3a708247559ad96, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:17:54,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/e284de4042414398a3a708247559ad96 is 175, key is test_row_0/A:col10/1731251872757/Put/seqid=0 2024-11-10T15:17:54,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742184_1360 (size=30955) 2024-11-10T15:17:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T15:17:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:54,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:17:54,981 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/e284de4042414398a3a708247559ad96 2024-11-10T15:17:54,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251934978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:54,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251934979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:54,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251934980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:54,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/706712ca6bbc47858f8b87de9d7bf4f1 is 50, key is test_row_0/B:col10/1731251872757/Put/seqid=0 2024-11-10T15:17:54,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251934981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:54,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251934982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:54,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:54,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:54,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:54,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742185_1361 (size=12001) 2024-11-10T15:17:54,998 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/706712ca6bbc47858f8b87de9d7bf4f1 2024-11-10T15:17:55,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/682748a4c2c9406992bcb2bd16f7589e is 50, key is test_row_0/C:col10/1731251872757/Put/seqid=0 2024-11-10T15:17:55,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742186_1362 (size=12001) 2024-11-10T15:17:55,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251935092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251935092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251935096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251935102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251935102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251935311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251935322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251935322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251935323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251935328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,420 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/682748a4c2c9406992bcb2bd16f7589e 2024-11-10T15:17:55,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/e284de4042414398a3a708247559ad96 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/e284de4042414398a3a708247559ad96 2024-11-10T15:17:55,432 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/e284de4042414398a3a708247559ad96, entries=150, sequenceid=36, filesize=30.2 K 2024-11-10T15:17:55,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/706712ca6bbc47858f8b87de9d7bf4f1 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/706712ca6bbc47858f8b87de9d7bf4f1 2024-11-10T15:17:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,441 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/706712ca6bbc47858f8b87de9d7bf4f1, entries=150, sequenceid=36, filesize=11.7 K 2024-11-10T15:17:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/682748a4c2c9406992bcb2bd16f7589e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/682748a4c2c9406992bcb2bd16f7589e 2024-11-10T15:17:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,446 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/682748a4c2c9406992bcb2bd16f7589e, entries=150, sequenceid=36, filesize=11.7 K 2024-11-10T15:17:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,447 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b423d13644c67f79070bddaad3c47156 in 998ms, sequenceid=36, compaction requested=false 2024-11-10T15:17:55,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:17:55,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:55,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-10T15:17:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-10T15:17:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-10T15:17:55,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7780 sec 2024-11-10T15:17:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.7870 sec 2024-11-10T15:17:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:55,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T15:17:55,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:17:55,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:55,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:17:55,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:55,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:17:55,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ebc326bcbef646b896ee8774d29308c5_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251874980/Put/seqid=0 2024-11-10T15:17:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742187_1363 (size=12154) 2024-11-10T15:17:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,752 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,757 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ebc326bcbef646b896ee8774d29308c5_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ebc326bcbef646b896ee8774d29308c5_b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,761 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/f769af09fa5d410a9bf2dca2ec9dd95b, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:17:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/f769af09fa5d410a9bf2dca2ec9dd95b is 175, key is test_row_0/A:col10/1731251874980/Put/seqid=0 2024-11-10T15:17:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:55,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742188_1364 (size=30955) 2024-11-10T15:17:55,818 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/f769af09fa5d410a9bf2dca2ec9dd95b 2024-11-10T15:17:55,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/f4fbc27a340e479c9697c03843d3936f is 50, key is test_row_0/B:col10/1731251874980/Put/seqid=0 2024-11-10T15:17:55,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251935823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251935824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251935829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251935833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251935865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742189_1365 (size=12001) 2024-11-10T15:17:55,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251935971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251935971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251935971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251935972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:56,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251935991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:56,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251936202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:56,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251936202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:56,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251936202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:56,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251936208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:56,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251936224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:56,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:56,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/f4fbc27a340e479c9697c03843d3936f 2024-11-10T15:17:56,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/6975eb0c7ba64c638696a65811d5ac61 is 50, key is test_row_0/C:col10/1731251874980/Put/seqid=0 2024-11-10T15:17:56,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742190_1366 (size=12001) 2024-11-10T15:17:56,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251936532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:56,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:56,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251936532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:56,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251936532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251936536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:56,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251936552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:56,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:56,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:56,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/6975eb0c7ba64c638696a65811d5ac61 2024-11-10T15:17:56,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/f769af09fa5d410a9bf2dca2ec9dd95b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/f769af09fa5d410a9bf2dca2ec9dd95b 2024-11-10T15:17:56,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/f769af09fa5d410a9bf2dca2ec9dd95b, entries=150, sequenceid=49, filesize=30.2 K 2024-11-10T15:17:56,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/f4fbc27a340e479c9697c03843d3936f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/f4fbc27a340e479c9697c03843d3936f 2024-11-10T15:17:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T15:17:56,837 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:56,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:17:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-10T15:17:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T15:17:56,849 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:17:56,856 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:17:56,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:17:56,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/f4fbc27a340e479c9697c03843d3936f, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T15:17:56,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/6975eb0c7ba64c638696a65811d5ac61 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/6975eb0c7ba64c638696a65811d5ac61 2024-11-10T15:17:56,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/6975eb0c7ba64c638696a65811d5ac61, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T15:17:56,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b423d13644c67f79070bddaad3c47156 in 1183ms, sequenceid=49, compaction requested=true 2024-11-10T15:17:56,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:17:56,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:17:56,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:56,886 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:56,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:17:56,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:56,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:17:56,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T15:17:56,887 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84271 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:56,887 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/A is initiating minor compaction (all files) 2024-11-10T15:17:56,888 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/A in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:56,888 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/a7b9ddb34c674c50bb8a8794e3ffbd13, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/e284de4042414398a3a708247559ad96, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/f769af09fa5d410a9bf2dca2ec9dd95b] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=82.3 K 2024-11-10T15:17:56,888 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:56,888 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/a7b9ddb34c674c50bb8a8794e3ffbd13, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/e284de4042414398a3a708247559ad96, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/f769af09fa5d410a9bf2dca2ec9dd95b] 2024-11-10T15:17:56,888 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:56,891 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting a7b9ddb34c674c50bb8a8794e3ffbd13, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731251872670 2024-11-10T15:17:56,892 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting e284de4042414398a3a708247559ad96, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731251872750 2024-11-10T15:17:56,892 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting f769af09fa5d410a9bf2dca2ec9dd95b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731251874979 2024-11-10T15:17:56,905 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:56,905 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/B is initiating minor compaction (all files) 2024-11-10T15:17:56,905 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/B in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:56,905 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/5a38518941ee48fda2bbd2ad29c25ccc, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/706712ca6bbc47858f8b87de9d7bf4f1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/f4fbc27a340e479c9697c03843d3936f] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=32.9 K 2024-11-10T15:17:56,906 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a38518941ee48fda2bbd2ad29c25ccc, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731251872670 2024-11-10T15:17:56,907 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 706712ca6bbc47858f8b87de9d7bf4f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731251872750 2024-11-10T15:17:56,908 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4fbc27a340e479c9697c03843d3936f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731251874979 2024-11-10T15:17:56,933 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:17:56,946 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110929ada5f358945f5a6f8ee4df6ab0a3f_b423d13644c67f79070bddaad3c47156 store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:17:56,948 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110929ada5f358945f5a6f8ee4df6ab0a3f_b423d13644c67f79070bddaad3c47156, store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:17:56,948 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110929ada5f358945f5a6f8ee4df6ab0a3f_b423d13644c67f79070bddaad3c47156 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:17:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T15:17:56,965 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#B#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:56,966 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/cd64bc77c22547749b306673c525929e is 50, key is test_row_0/B:col10/1731251874980/Put/seqid=0 2024-11-10T15:17:57,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742192_1368 (size=12104) 2024-11-10T15:17:57,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T15:17:57,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:57,013 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T15:17:57,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:17:57,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:57,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:17:57,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:57,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:17:57,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:57,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742191_1367 (size=4469) 2024-11-10T15:17:57,015 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/cd64bc77c22547749b306673c525929e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/cd64bc77c22547749b306673c525929e 2024-11-10T15:17:57,021 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/B of b423d13644c67f79070bddaad3c47156 into cd64bc77c22547749b306673c525929e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:57,021 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:17:57,022 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/B, priority=13, startTime=1731251876886; duration=0sec 2024-11-10T15:17:57,022 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:57,022 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:B 2024-11-10T15:17:57,022 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:57,029 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:57,029 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/C is initiating minor compaction (all files) 2024-11-10T15:17:57,029 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/C in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:57,029 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/9428dd32d4c6438b886a297eb4615e79, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/682748a4c2c9406992bcb2bd16f7589e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/6975eb0c7ba64c638696a65811d5ac61] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=32.9 K 2024-11-10T15:17:57,029 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9428dd32d4c6438b886a297eb4615e79, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731251872670 2024-11-10T15:17:57,030 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 682748a4c2c9406992bcb2bd16f7589e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731251872750 2024-11-10T15:17:57,030 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6975eb0c7ba64c638696a65811d5ac61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731251874979 2024-11-10T15:17:57,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107bd0996a3c4c47778eda1d44779a6dd6_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251875856/Put/seqid=0 2024-11-10T15:17:57,051 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#C#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:57,052 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/04a48bb966594780afbae3e058c933a3 is 50, key is test_row_0/C:col10/1731251874980/Put/seqid=0 2024-11-10T15:17:57,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:17:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:57,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742193_1369 (size=12154) 2024-11-10T15:17:57,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251937097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251937098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251937101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742194_1370 (size=12104) 2024-11-10T15:17:57,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251937111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251937113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T15:17:57,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251937221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251937221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251937221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251937243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251937243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,433 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#A#compaction#306 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:57,433 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/5269132e2ad247ec8a361d3aa47155ef is 175, key is test_row_0/A:col10/1731251874980/Put/seqid=0 2024-11-10T15:17:57,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742195_1371 (size=31058) 2024-11-10T15:17:57,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251937432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251937432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251937441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251937462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251937462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T15:17:57,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:57,512 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/5269132e2ad247ec8a361d3aa47155ef as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/5269132e2ad247ec8a361d3aa47155ef 2024-11-10T15:17:57,531 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107bd0996a3c4c47778eda1d44779a6dd6_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107bd0996a3c4c47778eda1d44779a6dd6_b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:57,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/01b2fb8995a94e3a8880981fb835c6a5, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:17:57,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/01b2fb8995a94e3a8880981fb835c6a5 is 175, key is test_row_0/A:col10/1731251875856/Put/seqid=0 2024-11-10T15:17:57,555 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/A of b423d13644c67f79070bddaad3c47156 into 5269132e2ad247ec8a361d3aa47155ef(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:57,555 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:17:57,555 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/A, priority=13, startTime=1731251876886; duration=0sec 2024-11-10T15:17:57,555 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:57,555 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:A 2024-11-10T15:17:57,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742196_1372 (size=30955) 2024-11-10T15:17:57,564 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/01b2fb8995a94e3a8880981fb835c6a5 2024-11-10T15:17:57,568 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/04a48bb966594780afbae3e058c933a3 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/04a48bb966594780afbae3e058c933a3 2024-11-10T15:17:57,592 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/C of b423d13644c67f79070bddaad3c47156 into 04a48bb966594780afbae3e058c933a3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:57,592 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:17:57,592 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/C, priority=13, startTime=1731251876886; duration=0sec 2024-11-10T15:17:57,593 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:57,593 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:C 2024-11-10T15:17:57,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/01485841a82e4bf2bc542b5e975a6da2 is 50, key is test_row_0/B:col10/1731251875856/Put/seqid=0 2024-11-10T15:17:57,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742197_1373 (size=12001) 2024-11-10T15:17:57,605 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/01485841a82e4bf2bc542b5e975a6da2 2024-11-10T15:17:57,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/cf406ba3b20749429dff4285d96e8504 is 50, key is test_row_0/C:col10/1731251875856/Put/seqid=0 2024-11-10T15:17:57,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742198_1374 (size=12001) 2024-11-10T15:17:57,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251937752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251937752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251937762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251937772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251937772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:57,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T15:17:58,044 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/cf406ba3b20749429dff4285d96e8504 2024-11-10T15:17:58,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/01b2fb8995a94e3a8880981fb835c6a5 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/01b2fb8995a94e3a8880981fb835c6a5 2024-11-10T15:17:58,094 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/01b2fb8995a94e3a8880981fb835c6a5, entries=150, sequenceid=68, filesize=30.2 K 2024-11-10T15:17:58,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/01485841a82e4bf2bc542b5e975a6da2 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/01485841a82e4bf2bc542b5e975a6da2 2024-11-10T15:17:58,101 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/01485841a82e4bf2bc542b5e975a6da2, entries=150, sequenceid=68, filesize=11.7 K 2024-11-10T15:17:58,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/cf406ba3b20749429dff4285d96e8504 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/cf406ba3b20749429dff4285d96e8504 2024-11-10T15:17:58,124 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/cf406ba3b20749429dff4285d96e8504, entries=150, sequenceid=68, filesize=11.7 K 2024-11-10T15:17:58,128 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b423d13644c67f79070bddaad3c47156 in 1115ms, sequenceid=68, compaction requested=false 2024-11-10T15:17:58,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:17:58,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:58,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-10T15:17:58,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-10T15:17:58,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-10T15:17:58,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2750 sec 2024-11-10T15:17:58,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.2980 sec 2024-11-10T15:17:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:58,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T15:17:58,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:17:58,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:58,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:17:58,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:58,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:17:58,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:58,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a33488c3593a4aa8a35a4fe4aad57f86_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251877110/Put/seqid=0 2024-11-10T15:17:58,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742199_1375 (size=14594) 2024-11-10T15:17:58,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251938333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:58,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251938346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:58,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:58,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251938353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:58,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:58,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251938361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:58,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251938361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:58,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251938474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:58,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251938474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:58,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251938474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:58,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:58,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:58,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251938481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:58,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:58,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251938493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:58,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:58,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251938691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:58,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251938692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:58,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251938693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:58,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251938702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:58,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:58,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251938711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:58,754 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:58,758 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a33488c3593a4aa8a35a4fe4aad57f86_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a33488c3593a4aa8a35a4fe4aad57f86_b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:58,761 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/dcec43efe0ef4808bad146ef82ee5033, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:17:58,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/dcec43efe0ef4808bad146ef82ee5033 is 175, key is test_row_0/A:col10/1731251877110/Put/seqid=0 2024-11-10T15:17:58,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742200_1376 (size=39549) 2024-11-10T15:17:58,792 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/dcec43efe0ef4808bad146ef82ee5033 2024-11-10T15:17:58,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/1ac0e648ec3d4259b8d05dc929be4240 is 50, key is test_row_0/B:col10/1731251877110/Put/seqid=0 2024-11-10T15:17:58,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742201_1377 (size=12001) 2024-11-10T15:17:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T15:17:58,992 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:17:58,993 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:17:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-10T15:17:58,995 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:17:58,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T15:17:58,996 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:17:58,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:17:59,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:59,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251939004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251939004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:59,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251939004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:59,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251939021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:59,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251939024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:59,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T15:17:59,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T15:17:59,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:59,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:17:59,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:59,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:59,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:59,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:59,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/1ac0e648ec3d4259b8d05dc929be4240 2024-11-10T15:17:59,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/99374be1314c4deba6959214d0bc6554 is 50, key is test_row_0/C:col10/1731251877110/Put/seqid=0 2024-11-10T15:17:59,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742202_1378 (size=12001) 2024-11-10T15:17:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T15:17:59,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T15:17:59,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:59,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:17:59,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:59,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:59,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:59,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T15:17:59,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:59,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:17:59,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:59,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:59,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:59,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251939522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:59,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:59,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251939524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:59,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:59,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251939533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:59,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:59,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251939552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:59,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:17:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251939554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:17:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:17:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:17:59,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T15:17:59,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T15:17:59,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:59,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:17:59,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:59,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:59,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:17:59,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/99374be1314c4deba6959214d0bc6554 2024-11-10T15:17:59,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/dcec43efe0ef4808bad146ef82ee5033 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/dcec43efe0ef4808bad146ef82ee5033 2024-11-10T15:17:59,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/dcec43efe0ef4808bad146ef82ee5033, entries=200, sequenceid=85, filesize=38.6 K 2024-11-10T15:17:59,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/1ac0e648ec3d4259b8d05dc929be4240 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/1ac0e648ec3d4259b8d05dc929be4240 2024-11-10T15:17:59,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/1ac0e648ec3d4259b8d05dc929be4240, entries=150, sequenceid=85, filesize=11.7 K 2024-11-10T15:17:59,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/99374be1314c4deba6959214d0bc6554 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/99374be1314c4deba6959214d0bc6554 2024-11-10T15:17:59,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/99374be1314c4deba6959214d0bc6554, entries=150, sequenceid=85, filesize=11.7 K 2024-11-10T15:17:59,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b423d13644c67f79070bddaad3c47156 in 1492ms, sequenceid=85, compaction requested=true 2024-11-10T15:17:59,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:17:59,765 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:59,768 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:59,768 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/A is initiating minor compaction (all files) 2024-11-10T15:17:59,768 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/A in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:59,768 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/5269132e2ad247ec8a361d3aa47155ef, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/01b2fb8995a94e3a8880981fb835c6a5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/dcec43efe0ef4808bad146ef82ee5033] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=99.2 K 2024-11-10T15:17:59,768 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:59,768 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/5269132e2ad247ec8a361d3aa47155ef, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/01b2fb8995a94e3a8880981fb835c6a5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/dcec43efe0ef4808bad146ef82ee5033] 2024-11-10T15:17:59,769 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5269132e2ad247ec8a361d3aa47155ef, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731251874979 2024-11-10T15:17:59,770 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01b2fb8995a94e3a8880981fb835c6a5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731251875792 2024-11-10T15:17:59,770 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcec43efe0ef4808bad146ef82ee5033, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731251877091 2024-11-10T15:17:59,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:17:59,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:59,776 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:59,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:17:59,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:17:59,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:17:59,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:59,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T15:17:59,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:59,783 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T15:17:59,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:17:59,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:59,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:17:59,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:59,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:17:59,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:17:59,784 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:17:59,784 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/B is initiating minor compaction (all files) 2024-11-10T15:17:59,784 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/B in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:17:59,784 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/cd64bc77c22547749b306673c525929e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/01485841a82e4bf2bc542b5e975a6da2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/1ac0e648ec3d4259b8d05dc929be4240] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=35.3 K 2024-11-10T15:17:59,786 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting cd64bc77c22547749b306673c525929e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731251874979 2024-11-10T15:17:59,786 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:17:59,787 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 01485841a82e4bf2bc542b5e975a6da2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731251875792 2024-11-10T15:17:59,788 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ac0e648ec3d4259b8d05dc929be4240, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731251877106 2024-11-10T15:17:59,801 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110fdb9506ded1c44cd9d0930cab879d855_b423d13644c67f79070bddaad3c47156 store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:17:59,803 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110fdb9506ded1c44cd9d0930cab879d855_b423d13644c67f79070bddaad3c47156, store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:17:59,803 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fdb9506ded1c44cd9d0930cab879d855_b423d13644c67f79070bddaad3c47156 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:17:59,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109258771336ee4f689d323bef40a416ca_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251878341/Put/seqid=0 2024-11-10T15:17:59,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742203_1379 (size=4469) 2024-11-10T15:17:59,883 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#A#compaction#315 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:59,883 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/469d4056b7a04949af886e0f1df0f5f6 is 175, key is test_row_0/A:col10/1731251877110/Put/seqid=0 2024-11-10T15:17:59,894 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#B#compaction#317 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:17:59,895 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/45f22cfc2cf44859a40470c82a0b7415 is 50, key is test_row_0/B:col10/1731251877110/Put/seqid=0 2024-11-10T15:17:59,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742204_1380 (size=12154) 2024-11-10T15:17:59,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:17:59,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742205_1381 (size=31161) 2024-11-10T15:17:59,965 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109258771336ee4f689d323bef40a416ca_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109258771336ee4f689d323bef40a416ca_b423d13644c67f79070bddaad3c47156 2024-11-10T15:17:59,965 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/469d4056b7a04949af886e0f1df0f5f6 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/469d4056b7a04949af886e0f1df0f5f6 2024-11-10T15:17:59,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/9516988daecb448284063fbd0208804b, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:17:59,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/9516988daecb448284063fbd0208804b is 175, key is test_row_0/A:col10/1731251878341/Put/seqid=0 2024-11-10T15:17:59,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742206_1382 (size=12207) 2024-11-10T15:17:59,982 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/A of b423d13644c67f79070bddaad3c47156 into 469d4056b7a04949af886e0f1df0f5f6(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:17:59,982 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:17:59,985 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/A, priority=13, startTime=1731251879765; duration=0sec 2024-11-10T15:17:59,985 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:17:59,989 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:A 2024-11-10T15:17:59,989 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:17:59,992 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/45f22cfc2cf44859a40470c82a0b7415 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/45f22cfc2cf44859a40470c82a0b7415 2024-11-10T15:17:59,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742207_1383 (size=30955) 2024-11-10T15:18:00,004 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/9516988daecb448284063fbd0208804b 2024-11-10T15:18:00,007 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:00,007 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/C is initiating minor compaction (all files) 2024-11-10T15:18:00,007 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/C in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:00,007 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/04a48bb966594780afbae3e058c933a3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/cf406ba3b20749429dff4285d96e8504, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/99374be1314c4deba6959214d0bc6554] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=35.3 K 2024-11-10T15:18:00,009 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/B of b423d13644c67f79070bddaad3c47156 into 45f22cfc2cf44859a40470c82a0b7415(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:00,009 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:00,009 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/B, priority=13, startTime=1731251879776; duration=0sec 2024-11-10T15:18:00,009 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:00,009 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:B 2024-11-10T15:18:00,015 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04a48bb966594780afbae3e058c933a3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731251874979 2024-11-10T15:18:00,017 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf406ba3b20749429dff4285d96e8504, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731251875792 2024-11-10T15:18:00,019 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99374be1314c4deba6959214d0bc6554, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731251877106 2024-11-10T15:18:00,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/0b3169639973491384265dce92c23e0b is 50, key is test_row_0/B:col10/1731251878341/Put/seqid=0 2024-11-10T15:18:00,063 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#C#compaction#319 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:00,064 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/28d67a32db844c7a9220c1d82468fb35 is 50, key is test_row_0/C:col10/1731251877110/Put/seqid=0 2024-11-10T15:18:00,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742208_1384 (size=12001) 2024-11-10T15:18:00,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742209_1385 (size=12207) 2024-11-10T15:18:00,085 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/0b3169639973491384265dce92c23e0b 2024-11-10T15:18:00,101 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/28d67a32db844c7a9220c1d82468fb35 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/28d67a32db844c7a9220c1d82468fb35 2024-11-10T15:18:00,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/1d46dae6a32a46c3b5ed3d292e9038bf is 50, key is test_row_0/C:col10/1731251878341/Put/seqid=0 2024-11-10T15:18:00,131 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/C of b423d13644c67f79070bddaad3c47156 into 28d67a32db844c7a9220c1d82468fb35(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:00,131 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:00,131 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/C, priority=13, startTime=1731251879777; duration=0sec 2024-11-10T15:18:00,131 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:00,131 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:C 2024-11-10T15:18:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T15:18:00,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742210_1386 (size=12001) 2024-11-10T15:18:00,150 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/1d46dae6a32a46c3b5ed3d292e9038bf 2024-11-10T15:18:00,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/9516988daecb448284063fbd0208804b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/9516988daecb448284063fbd0208804b 2024-11-10T15:18:00,166 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/9516988daecb448284063fbd0208804b, entries=150, sequenceid=103, filesize=30.2 K 2024-11-10T15:18:00,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/0b3169639973491384265dce92c23e0b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/0b3169639973491384265dce92c23e0b 2024-11-10T15:18:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,172 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/0b3169639973491384265dce92c23e0b, entries=150, sequenceid=103, filesize=11.7 K 2024-11-10T15:18:00,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/1d46dae6a32a46c3b5ed3d292e9038bf as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/1d46dae6a32a46c3b5ed3d292e9038bf 2024-11-10T15:18:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,179 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/1d46dae6a32a46c3b5ed3d292e9038bf, entries=150, sequenceid=103, filesize=11.7 K 2024-11-10T15:18:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,181 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for b423d13644c67f79070bddaad3c47156 in 398ms, sequenceid=103, compaction requested=false 2024-11-10T15:18:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:00,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:00,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-10T15:18:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-10T15:18:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-10T15:18:00,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1860 sec 2024-11-10T15:18:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.1940 sec 2024-11-10T15:18:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T15:18:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:18:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:18:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:18:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d9a56d61febb4040989a6f63285aa329_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251880734/Put/seqid=0 2024-11-10T15:18:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:00,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742211_1387 (size=19474) 2024-11-10T15:18:00,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731251880830, max timestamp=1731251880933Num uniques 859; 2024-11-10T15:18:00,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731251880830, max timestamp=1731251880933Num uniques 859; 2024-11-10T15:18:00,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T15:18:00,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T15:18:00,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731251880830, max timestamp=1731251880933Num uniques 859; 2024-11-10T15:18:00,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T15:18:00,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251940984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251940986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251940987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251940989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251940990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251941102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251941111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251941112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251941112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251941112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T15:18:01,142 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:01,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-10T15:18:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T15:18:01,145 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:01,146 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:01,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:01,241 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:01,247 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d9a56d61febb4040989a6f63285aa329_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d9a56d61febb4040989a6f63285aa329_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:01,248 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/38609d6211bd4580a0e4155ccfb201e8, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:01,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/38609d6211bd4580a0e4155ccfb201e8 is 175, key is test_row_0/A:col10/1731251880734/Put/seqid=0 2024-11-10T15:18:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T15:18:01,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742212_1388 (size=56733) 2024-11-10T15:18:01,277 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/38609d6211bd4580a0e4155ccfb201e8 2024-11-10T15:18:01,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/b42585cb30b547079ae25ce758b50c37 is 50, key is test_row_0/B:col10/1731251880734/Put/seqid=0 2024-11-10T15:18:01,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T15:18:01,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:01,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:01,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:01,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:01,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:01,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742213_1389 (size=12001) 2024-11-10T15:18:01,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251941321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251941322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251941322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251941322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/b42585cb30b547079ae25ce758b50c37 2024-11-10T15:18:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251941322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/46260ac2490d43029ef4248a3ab6708d is 50, key is test_row_0/C:col10/1731251880734/Put/seqid=0 2024-11-10T15:18:01,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742214_1390 (size=12001) 2024-11-10T15:18:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T15:18:01,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T15:18:01,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:01,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:01,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:01,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:01,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:01,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T15:18:01,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:01,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:01,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:01,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:01,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:01,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251941641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251941642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251941642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251941642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251941642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:01,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/46260ac2490d43029ef4248a3ab6708d 2024-11-10T15:18:01,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T15:18:01,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:01,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T15:18:01,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:01,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:01,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:01,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/38609d6211bd4580a0e4155ccfb201e8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/38609d6211bd4580a0e4155ccfb201e8 2024-11-10T15:18:01,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/38609d6211bd4580a0e4155ccfb201e8, entries=300, sequenceid=116, filesize=55.4 K 2024-11-10T15:18:01,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/b42585cb30b547079ae25ce758b50c37 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/b42585cb30b547079ae25ce758b50c37 2024-11-10T15:18:01,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/b42585cb30b547079ae25ce758b50c37, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T15:18:01,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/46260ac2490d43029ef4248a3ab6708d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/46260ac2490d43029ef4248a3ab6708d 2024-11-10T15:18:01,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/46260ac2490d43029ef4248a3ab6708d, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T15:18:01,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b423d13644c67f79070bddaad3c47156 in 1075ms, sequenceid=116, compaction requested=true 2024-11-10T15:18:01,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:01,813 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:01,814 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118849 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:01,814 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/A is initiating minor compaction (all files) 2024-11-10T15:18:01,814 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/A in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:01,814 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/469d4056b7a04949af886e0f1df0f5f6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/9516988daecb448284063fbd0208804b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/38609d6211bd4580a0e4155ccfb201e8] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=116.1 K 2024-11-10T15:18:01,814 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:01,814 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/469d4056b7a04949af886e0f1df0f5f6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/9516988daecb448284063fbd0208804b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/38609d6211bd4580a0e4155ccfb201e8] 2024-11-10T15:18:01,815 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 469d4056b7a04949af886e0f1df0f5f6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731251877106 2024-11-10T15:18:01,815 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9516988daecb448284063fbd0208804b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731251878341 2024-11-10T15:18:01,815 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38609d6211bd4580a0e4155ccfb201e8, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731251880661 2024-11-10T15:18:01,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:01,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:01,824 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:01,826 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:01,826 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/B is initiating minor compaction (all files) 2024-11-10T15:18:01,827 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/B in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:01,827 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/45f22cfc2cf44859a40470c82a0b7415, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/0b3169639973491384265dce92c23e0b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/b42585cb30b547079ae25ce758b50c37] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=35.4 K 2024-11-10T15:18:01,828 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 45f22cfc2cf44859a40470c82a0b7415, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731251877106 2024-11-10T15:18:01,828 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b3169639973491384265dce92c23e0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731251878341 2024-11-10T15:18:01,828 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting b42585cb30b547079ae25ce758b50c37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731251880724 2024-11-10T15:18:01,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:01,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:01,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:01,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:01,837 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:01,842 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#B#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:01,843 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/262618b24cfc419abf4d56d430d5d26d is 50, key is test_row_0/B:col10/1731251880734/Put/seqid=0 2024-11-10T15:18:01,845 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411106e2aed36353c4b5998a9c8eb5998655e_b423d13644c67f79070bddaad3c47156 store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:01,848 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411106e2aed36353c4b5998a9c8eb5998655e_b423d13644c67f79070bddaad3c47156, store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:01,848 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106e2aed36353c4b5998a9c8eb5998655e_b423d13644c67f79070bddaad3c47156 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:01,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742216_1392 (size=4469) 2024-11-10T15:18:01,893 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#A#compaction#324 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:01,894 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/dfc6ab5c3e3c45718530d2866f117522 is 175, key is test_row_0/A:col10/1731251880734/Put/seqid=0 2024-11-10T15:18:01,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742215_1391 (size=12309) 2024-11-10T15:18:01,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T15:18:01,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:01,935 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-10T15:18:01,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:18:01,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:01,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:18:01,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:01,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:18:01,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:01,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742217_1393 (size=31263) 2024-11-10T15:18:01,974 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/dfc6ab5c3e3c45718530d2866f117522 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/dfc6ab5c3e3c45718530d2866f117522 2024-11-10T15:18:01,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110774c81abd7314acbaeaeca4f7841df58_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251880989/Put/seqid=0 2024-11-10T15:18:01,989 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/A of b423d13644c67f79070bddaad3c47156 into dfc6ab5c3e3c45718530d2866f117522(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:01,989 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:01,989 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/A, priority=13, startTime=1731251881813; duration=0sec 2024-11-10T15:18:01,989 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:01,989 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:A 2024-11-10T15:18:01,989 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:01,990 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:01,990 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/C is initiating minor compaction (all files) 2024-11-10T15:18:01,991 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/C in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:01,991 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/28d67a32db844c7a9220c1d82468fb35, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/1d46dae6a32a46c3b5ed3d292e9038bf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/46260ac2490d43029ef4248a3ab6708d] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=35.4 K 2024-11-10T15:18:01,991 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28d67a32db844c7a9220c1d82468fb35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731251877106 2024-11-10T15:18:01,991 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d46dae6a32a46c3b5ed3d292e9038bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731251878341 2024-11-10T15:18:01,992 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46260ac2490d43029ef4248a3ab6708d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731251880724 2024-11-10T15:18:02,007 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:02,007 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/4e4eadd0971d4517a2b750a9742b21f8 is 50, key is test_row_0/C:col10/1731251880734/Put/seqid=0 2024-11-10T15:18:02,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742218_1394 (size=12304) 2024-11-10T15:18:02,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,014 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110774c81abd7314acbaeaeca4f7841df58_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110774c81abd7314acbaeaeca4f7841df58_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:02,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/0ca68244fc234418bacb11e8ff4ef0ce, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:02,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/0ca68244fc234418bacb11e8ff4ef0ce is 175, key is test_row_0/A:col10/1731251880989/Put/seqid=0 2024-11-10T15:18:02,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742220_1396 (size=31105) 2024-11-10T15:18:02,046 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/0ca68244fc234418bacb11e8ff4ef0ce 2024-11-10T15:18:02,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742219_1395 (size=12309) 2024-11-10T15:18:02,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/aca8a2a3789b4eb5bd51f6c5c1d5e2da is 50, key is test_row_0/B:col10/1731251880989/Put/seqid=0 2024-11-10T15:18:02,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742221_1397 (size=12151) 2024-11-10T15:18:02,115 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/aca8a2a3789b4eb5bd51f6c5c1d5e2da 2024-11-10T15:18:02,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/ec1b4b410b95413e87a296e5ce4d3d7a is 50, key is test_row_0/C:col10/1731251880989/Put/seqid=0 2024-11-10T15:18:02,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742222_1398 (size=12151) 2024-11-10T15:18:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:02,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:02,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:02,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251942191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:02,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:02,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251942212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:02,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251942214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:02,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251942214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:02,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251942217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T15:18:02,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251942322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:02,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:02,342 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/262618b24cfc419abf4d56d430d5d26d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/262618b24cfc419abf4d56d430d5d26d 2024-11-10T15:18:02,347 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/B of b423d13644c67f79070bddaad3c47156 into 262618b24cfc419abf4d56d430d5d26d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:02,347 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:02,348 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/B, priority=13, startTime=1731251881824; duration=0sec 2024-11-10T15:18:02,348 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:02,348 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:B 2024-11-10T15:18:02,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251942341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:02,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251942341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:02,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251942341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:02,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:02,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251942343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:02,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:02,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:02,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:02,464 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/4e4eadd0971d4517a2b750a9742b21f8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/4e4eadd0971d4517a2b750a9742b21f8 2024-11-10T15:18:02,472 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/C of b423d13644c67f79070bddaad3c47156 into 4e4eadd0971d4517a2b750a9742b21f8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:02,472 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:02,472 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/C, priority=13, startTime=1731251881835; duration=0sec 2024-11-10T15:18:02,472 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:02,472 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:C 2024-11-10T15:18:02,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:02,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251942542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:02,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:02,562 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/ec1b4b410b95413e87a296e5ce4d3d7a 2024-11-10T15:18:02,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:02,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251942562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:02,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251942562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:02,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251942562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251942562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:02,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/0ca68244fc234418bacb11e8ff4ef0ce as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/0ca68244fc234418bacb11e8ff4ef0ce 2024-11-10T15:18:02,573 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/0ca68244fc234418bacb11e8ff4ef0ce, entries=150, sequenceid=151, filesize=30.4 K 2024-11-10T15:18:02,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/aca8a2a3789b4eb5bd51f6c5c1d5e2da as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/aca8a2a3789b4eb5bd51f6c5c1d5e2da 2024-11-10T15:18:02,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,580 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/aca8a2a3789b4eb5bd51f6c5c1d5e2da, entries=150, sequenceid=151, filesize=11.9 K 2024-11-10T15:18:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/ec1b4b410b95413e87a296e5ce4d3d7a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/ec1b4b410b95413e87a296e5ce4d3d7a 2024-11-10T15:18:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,586 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/ec1b4b410b95413e87a296e5ce4d3d7a, entries=150, sequenceid=151, filesize=11.9 K 2024-11-10T15:18:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,587 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=60.38 KB/61830 for b423d13644c67f79070bddaad3c47156 in 651ms, sequenceid=151, compaction requested=false 2024-11-10T15:18:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:02,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:02,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-10T15:18:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-10T15:18:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-10T15:18:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4420 sec 2024-11-10T15:18:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,593 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.4480 sec 2024-11-10T15:18:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T15:18:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:18:02,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:02,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:18:02,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:02,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:18:02,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:02,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111037db3eb39b4645aab064c52d1136c13b_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251882208/Put/seqid=0 2024-11-10T15:18:03,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742223_1399 (size=12304) 2024-11-10T15:18:03,023 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:03,027 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111037db3eb39b4645aab064c52d1136c13b_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111037db3eb39b4645aab064c52d1136c13b_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:03,028 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/1adce74f432745ff98cb76e4d7ae3f82, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:03,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/1adce74f432745ff98cb76e4d7ae3f82 is 175, key is test_row_0/A:col10/1731251882208/Put/seqid=0 2024-11-10T15:18:03,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251943022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251943034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251943036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251943037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251943038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742224_1400 (size=31105) 2024-11-10T15:18:03,085 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/1adce74f432745ff98cb76e4d7ae3f82 2024-11-10T15:18:03,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/0e39a49eb5da43b0811fe9a8e3ce6f9a is 50, key is test_row_0/B:col10/1731251882208/Put/seqid=0 2024-11-10T15:18:03,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742225_1401 (size=12151) 2024-11-10T15:18:03,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251943161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251943161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251943161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251943161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251943161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T15:18:03,292 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:03,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-10T15:18:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T15:18:03,294 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:03,296 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:03,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:03,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251943381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251943381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251943381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251943381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251943382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T15:18:03,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T15:18:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:03,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:03,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:03,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/0e39a49eb5da43b0811fe9a8e3ce6f9a 2024-11-10T15:18:03,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/9257842fa5a64ccd977be9194c719b46 is 50, key is test_row_0/C:col10/1731251882208/Put/seqid=0 2024-11-10T15:18:03,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742226_1402 (size=12151) 2024-11-10T15:18:03,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T15:18:03,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:03,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:03,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:03,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:03,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T15:18:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:03,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251943702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251943702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251943702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251943702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251943704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:03,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T15:18:03,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:03,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:03,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:03,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:03,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:03,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T15:18:03,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:03,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:03,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:03,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:03,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T15:18:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:03,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/9257842fa5a64ccd977be9194c719b46 2024-11-10T15:18:04,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/1adce74f432745ff98cb76e4d7ae3f82 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/1adce74f432745ff98cb76e4d7ae3f82 2024-11-10T15:18:04,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/1adce74f432745ff98cb76e4d7ae3f82, entries=150, sequenceid=167, filesize=30.4 K 2024-11-10T15:18:04,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/0e39a49eb5da43b0811fe9a8e3ce6f9a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/0e39a49eb5da43b0811fe9a8e3ce6f9a 2024-11-10T15:18:04,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/0e39a49eb5da43b0811fe9a8e3ce6f9a, entries=150, sequenceid=167, filesize=11.9 K 2024-11-10T15:18:04,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/9257842fa5a64ccd977be9194c719b46 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/9257842fa5a64ccd977be9194c719b46 2024-11-10T15:18:04,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/9257842fa5a64ccd977be9194c719b46, entries=150, sequenceid=167, filesize=11.9 K 2024-11-10T15:18:04,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b423d13644c67f79070bddaad3c47156 in 1080ms, sequenceid=167, compaction requested=true 2024-11-10T15:18:04,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:04,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:04,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:04,035 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:04,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:04,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:04,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:04,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T15:18:04,037 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93473 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:04,037 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/A is initiating minor compaction (all files) 2024-11-10T15:18:04,038 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/A in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:04,038 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/dfc6ab5c3e3c45718530d2866f117522, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/0ca68244fc234418bacb11e8ff4ef0ce, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/1adce74f432745ff98cb76e4d7ae3f82] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=91.3 K 2024-11-10T15:18:04,038 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:04,038 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/dfc6ab5c3e3c45718530d2866f117522, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/0ca68244fc234418bacb11e8ff4ef0ce, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/1adce74f432745ff98cb76e4d7ae3f82] 2024-11-10T15:18:04,038 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfc6ab5c3e3c45718530d2866f117522, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731251880724 2024-11-10T15:18:04,039 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ca68244fc234418bacb11e8ff4ef0ce, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731251880985 2024-11-10T15:18:04,039 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1adce74f432745ff98cb76e4d7ae3f82, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731251882208 2024-11-10T15:18:04,040 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:04,047 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:04,047 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/B is initiating minor compaction (all files) 2024-11-10T15:18:04,047 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/B in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:04,047 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/262618b24cfc419abf4d56d430d5d26d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/aca8a2a3789b4eb5bd51f6c5c1d5e2da, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/0e39a49eb5da43b0811fe9a8e3ce6f9a] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=35.8 K 2024-11-10T15:18:04,048 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 262618b24cfc419abf4d56d430d5d26d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731251880724 2024-11-10T15:18:04,048 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting aca8a2a3789b4eb5bd51f6c5c1d5e2da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731251880985 2024-11-10T15:18:04,049 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e39a49eb5da43b0811fe9a8e3ce6f9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731251882208 2024-11-10T15:18:04,058 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:04,069 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111001c9f8fed9da421795ac9f0097fa55fc_b423d13644c67f79070bddaad3c47156 store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:04,072 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111001c9f8fed9da421795ac9f0097fa55fc_b423d13644c67f79070bddaad3c47156, store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:04,072 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111001c9f8fed9da421795ac9f0097fa55fc_b423d13644c67f79070bddaad3c47156 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:04,081 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#B#compaction#334 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:04,081 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/024459843af8418babc2889715f43585 is 50, key is test_row_0/B:col10/1731251882208/Put/seqid=0 2024-11-10T15:18:04,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T15:18:04,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:04,088 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T15:18:04,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:18:04,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:04,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:18:04,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:04,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:18:04,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:04,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742227_1403 (size=4469) 2024-11-10T15:18:04,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742228_1404 (size=12561) 2024-11-10T15:18:04,153 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/024459843af8418babc2889715f43585 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/024459843af8418babc2889715f43585 2024-11-10T15:18:04,160 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/B of b423d13644c67f79070bddaad3c47156 into 024459843af8418babc2889715f43585(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:04,160 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:04,160 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/B, priority=13, startTime=1731251884035; duration=0sec 2024-11-10T15:18:04,160 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:04,160 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:B 2024-11-10T15:18:04,160 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:04,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a25b5679834545ba9e4527ee81604921_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251883036/Put/seqid=0 2024-11-10T15:18:04,173 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:04,173 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/C is initiating minor compaction (all files) 2024-11-10T15:18:04,173 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/C in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:04,173 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/4e4eadd0971d4517a2b750a9742b21f8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/ec1b4b410b95413e87a296e5ce4d3d7a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/9257842fa5a64ccd977be9194c719b46] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=35.8 K 2024-11-10T15:18:04,176 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e4eadd0971d4517a2b750a9742b21f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731251880724 2024-11-10T15:18:04,180 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting ec1b4b410b95413e87a296e5ce4d3d7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731251880985 2024-11-10T15:18:04,182 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9257842fa5a64ccd977be9194c719b46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731251882208 2024-11-10T15:18:04,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742229_1405 (size=12304) 2024-11-10T15:18:04,209 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#C#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:04,209 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/854f9c49e418416e95aec5ad03e48d0b is 50, key is test_row_0/C:col10/1731251882208/Put/seqid=0 2024-11-10T15:18:04,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:04,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:04,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:04,247 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a25b5679834545ba9e4527ee81604921_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a25b5679834545ba9e4527ee81604921_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:04,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742230_1406 (size=12561) 2024-11-10T15:18:04,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/000098616c4f4a57b2a8cabcb94361e0, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:04,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/000098616c4f4a57b2a8cabcb94361e0 is 175, key is test_row_0/A:col10/1731251883036/Put/seqid=0 2024-11-10T15:18:04,294 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/854f9c49e418416e95aec5ad03e48d0b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/854f9c49e418416e95aec5ad03e48d0b 2024-11-10T15:18:04,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251944277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251944289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742231_1407 (size=31105) 2024-11-10T15:18:04,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251944296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251944296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251944297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,329 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/C of b423d13644c67f79070bddaad3c47156 into 854f9c49e418416e95aec5ad03e48d0b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:04,329 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:04,329 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/C, priority=13, startTime=1731251884035; duration=0sec 2024-11-10T15:18:04,329 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:04,329 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:C 2024-11-10T15:18:04,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251944412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251944422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251944422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T15:18:04,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251944432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251944437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,533 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#A#compaction#333 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:04,534 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/c3363eacf3f244bab8ab4b4d8c89a666 is 175, key is test_row_0/A:col10/1731251882208/Put/seqid=0 2024-11-10T15:18:04,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742232_1408 (size=31515) 2024-11-10T15:18:04,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251944632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251944632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251944642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251944642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251944661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,720 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/000098616c4f4a57b2a8cabcb94361e0 2024-11-10T15:18:04,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/85908f605e944d25873be4238aedbf07 is 50, key is test_row_0/B:col10/1731251883036/Put/seqid=0 2024-11-10T15:18:04,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742233_1409 (size=12151) 2024-11-10T15:18:04,794 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/85908f605e944d25873be4238aedbf07 2024-11-10T15:18:04,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/b988868064c44ce28e8c149fdd6b89d6 is 50, key is test_row_0/C:col10/1731251883036/Put/seqid=0 2024-11-10T15:18:04,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742234_1410 (size=12151) 2024-11-10T15:18:04,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251944951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251944952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251944962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251944962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:04,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251944981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:05,017 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/c3363eacf3f244bab8ab4b4d8c89a666 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/c3363eacf3f244bab8ab4b4d8c89a666 2024-11-10T15:18:05,029 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/A of b423d13644c67f79070bddaad3c47156 into c3363eacf3f244bab8ab4b4d8c89a666(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:05,029 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:05,029 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/A, priority=13, startTime=1731251884035; duration=0sec 2024-11-10T15:18:05,029 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:05,029 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:A 2024-11-10T15:18:05,247 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/b988868064c44ce28e8c149fdd6b89d6 2024-11-10T15:18:05,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/000098616c4f4a57b2a8cabcb94361e0 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/000098616c4f4a57b2a8cabcb94361e0 2024-11-10T15:18:05,265 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/000098616c4f4a57b2a8cabcb94361e0, entries=150, sequenceid=186, filesize=30.4 K 2024-11-10T15:18:05,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/85908f605e944d25873be4238aedbf07 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/85908f605e944d25873be4238aedbf07 2024-11-10T15:18:05,277 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/85908f605e944d25873be4238aedbf07, entries=150, sequenceid=186, filesize=11.9 K 2024-11-10T15:18:05,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/b988868064c44ce28e8c149fdd6b89d6 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/b988868064c44ce28e8c149fdd6b89d6 2024-11-10T15:18:05,283 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/b988868064c44ce28e8c149fdd6b89d6, entries=150, sequenceid=186, filesize=11.9 K 2024-11-10T15:18:05,285 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for b423d13644c67f79070bddaad3c47156 in 1197ms, sequenceid=186, compaction requested=false 2024-11-10T15:18:05,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:05,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:05,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-10T15:18:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-10T15:18:05,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-10T15:18:05,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9910 sec 2024-11-10T15:18:05,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.0030 sec 2024-11-10T15:18:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T15:18:05,441 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:05,442 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-10T15:18:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T15:18:05,444 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:05,445 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:05,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:05,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T15:18:05,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:18:05,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:05,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:18:05,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:05,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:18:05,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:05,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111027a9cdd74e7e4fefa880f829c5e5e00e_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251884291/Put/seqid=0 2024-11-10T15:18:05,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742235_1411 (size=14794) 2024-11-10T15:18:05,551 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T15:18:05,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251945536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:05,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251945538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:05,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:05,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251945540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:05,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251945544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:05,558 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111027a9cdd74e7e4fefa880f829c5e5e00e_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111027a9cdd74e7e4fefa880f829c5e5e00e_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:05,559 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/38ce898e80e64e1c89fe528a5b0eae53, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:05,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/38ce898e80e64e1c89fe528a5b0eae53 is 175, key is test_row_0/A:col10/1731251884291/Put/seqid=0 2024-11-10T15:18:05,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251945554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:05,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742236_1412 (size=39749) 2024-11-10T15:18:05,579 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/38ce898e80e64e1c89fe528a5b0eae53 2024-11-10T15:18:05,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T15:18:05,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:05,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:05,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:05,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:05,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:05,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/11ea648d288c4329b17d314b8192e353 is 50, key is test_row_0/B:col10/1731251884291/Put/seqid=0 2024-11-10T15:18:05,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742237_1413 (size=12151) 2024-11-10T15:18:05,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251945665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:05,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251945665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:05,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251945665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:05,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251945665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:05,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251945682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:05,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:05,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T15:18:05,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:05,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:05,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:05,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:05,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T15:18:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:05,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251945892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251945892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:05,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251945892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:05,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251945892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:05,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T15:18:05,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:05,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:05,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:05,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:05,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:05,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251945912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:05,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:05,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:06,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/11ea648d288c4329b17d314b8192e353 2024-11-10T15:18:06,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/b839c9b13a904f329be2dce822830938 is 50, key is test_row_0/C:col10/1731251884291/Put/seqid=0 2024-11-10T15:18:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T15:18:06,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742238_1414 (size=12151) 2024-11-10T15:18:06,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T15:18:06,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:06,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:06,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:06,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:06,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:06,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251946212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:06,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251946212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:06,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251946212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251946212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:06,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T15:18:06,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:06,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:06,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:06,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:06,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:06,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251946241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:06,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:06,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T15:18:06,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:06,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:06,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:06,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:06,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:06,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/b839c9b13a904f329be2dce822830938 2024-11-10T15:18:06,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/38ce898e80e64e1c89fe528a5b0eae53 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/38ce898e80e64e1c89fe528a5b0eae53 2024-11-10T15:18:06,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/38ce898e80e64e1c89fe528a5b0eae53, entries=200, sequenceid=204, filesize=38.8 K 2024-11-10T15:18:06,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/11ea648d288c4329b17d314b8192e353 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/11ea648d288c4329b17d314b8192e353 2024-11-10T15:18:06,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/11ea648d288c4329b17d314b8192e353, entries=150, sequenceid=204, filesize=11.9 K 2024-11-10T15:18:06,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/b839c9b13a904f329be2dce822830938 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/b839c9b13a904f329be2dce822830938 2024-11-10T15:18:06,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/b839c9b13a904f329be2dce822830938, entries=150, sequenceid=204, filesize=11.9 K 2024-11-10T15:18:06,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b423d13644c67f79070bddaad3c47156 in 1020ms, sequenceid=204, compaction requested=true 2024-11-10T15:18:06,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:06,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:06,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:06,496 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:06,496 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:06,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:06,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:06,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:06,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:06,497 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:06,497 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/B is initiating minor compaction (all files) 2024-11-10T15:18:06,497 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/B in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:06,497 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/024459843af8418babc2889715f43585, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/85908f605e944d25873be4238aedbf07, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/11ea648d288c4329b17d314b8192e353] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=36.0 K 2024-11-10T15:18:06,497 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:06,497 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/A is initiating minor compaction (all files) 2024-11-10T15:18:06,497 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/A in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:06,498 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/c3363eacf3f244bab8ab4b4d8c89a666, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/000098616c4f4a57b2a8cabcb94361e0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/38ce898e80e64e1c89fe528a5b0eae53] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=100.0 K 2024-11-10T15:18:06,498 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:06,498 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/c3363eacf3f244bab8ab4b4d8c89a666, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/000098616c4f4a57b2a8cabcb94361e0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/38ce898e80e64e1c89fe528a5b0eae53] 2024-11-10T15:18:06,498 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 024459843af8418babc2889715f43585, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731251882208 2024-11-10T15:18:06,498 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3363eacf3f244bab8ab4b4d8c89a666, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731251882208 2024-11-10T15:18:06,499 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 85908f605e944d25873be4238aedbf07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731251882985 2024-11-10T15:18:06,499 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 000098616c4f4a57b2a8cabcb94361e0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731251882985 2024-11-10T15:18:06,499 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 11ea648d288c4329b17d314b8192e353, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731251884271 2024-11-10T15:18:06,499 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38ce898e80e64e1c89fe528a5b0eae53, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731251884271 2024-11-10T15:18:06,506 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:06,507 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/6b3bc412d8a84fd79d6b815f6fd99de1 is 50, key is test_row_0/B:col10/1731251884291/Put/seqid=0 2024-11-10T15:18:06,509 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:06,517 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411106f108128cf8a4af9b94194945662fe8f_b423d13644c67f79070bddaad3c47156 store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:06,519 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411106f108128cf8a4af9b94194945662fe8f_b423d13644c67f79070bddaad3c47156, store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:06,519 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106f108128cf8a4af9b94194945662fe8f_b423d13644c67f79070bddaad3c47156 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:06,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742239_1415 (size=12663) 2024-11-10T15:18:06,523 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/6b3bc412d8a84fd79d6b815f6fd99de1 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/6b3bc412d8a84fd79d6b815f6fd99de1 2024-11-10T15:18:06,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742240_1416 (size=4469) 2024-11-10T15:18:06,527 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/B of b423d13644c67f79070bddaad3c47156 into 6b3bc412d8a84fd79d6b815f6fd99de1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:06,527 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:06,527 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/B, priority=13, startTime=1731251886496; duration=0sec 2024-11-10T15:18:06,527 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:06,527 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:B 2024-11-10T15:18:06,527 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:06,528 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:06,528 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/C is initiating minor compaction (all files) 2024-11-10T15:18:06,528 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/C in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:06,529 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/854f9c49e418416e95aec5ad03e48d0b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/b988868064c44ce28e8c149fdd6b89d6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/b839c9b13a904f329be2dce822830938] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=36.0 K 2024-11-10T15:18:06,529 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 854f9c49e418416e95aec5ad03e48d0b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731251882208 2024-11-10T15:18:06,531 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting b988868064c44ce28e8c149fdd6b89d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731251882985 2024-11-10T15:18:06,531 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting b839c9b13a904f329be2dce822830938, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731251884271 2024-11-10T15:18:06,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T15:18:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:06,537 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T15:18:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:18:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:18:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:18:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:06,542 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#C#compaction#344 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:06,542 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/ef6cd149fdd54c75a773529791fedbee is 50, key is test_row_0/C:col10/1731251884291/Put/seqid=0 2024-11-10T15:18:06,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b1624a3258c140b8ac139d9efeaaf83d_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251885536/Put/seqid=0 2024-11-10T15:18:06,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742241_1417 (size=12663) 2024-11-10T15:18:06,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742242_1418 (size=12304) 2024-11-10T15:18:06,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:06,568 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b1624a3258c140b8ac139d9efeaaf83d_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b1624a3258c140b8ac139d9efeaaf83d_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:06,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/5f19e9e30303400fabbce00d3ae082e8, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:06,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/5f19e9e30303400fabbce00d3ae082e8 is 175, key is test_row_0/A:col10/1731251885536/Put/seqid=0 2024-11-10T15:18:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T15:18:06,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742243_1419 (size=31105) 2024-11-10T15:18:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:06,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:06,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251946785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:06,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251946799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:06,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251946800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:06,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:06,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251946802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251946803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:06,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251946911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251946911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:06,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:06,928 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#A#compaction#343 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:06,928 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/2d495234b5aa44289364835ace87d7cf is 175, key is test_row_0/A:col10/1731251884291/Put/seqid=0 2024-11-10T15:18:06,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:06,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251946922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251946922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:06,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251946922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:06,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:06,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:06,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742244_1420 (size=31617) 2024-11-10T15:18:06,958 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/ef6cd149fdd54c75a773529791fedbee as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/ef6cd149fdd54c75a773529791fedbee 2024-11-10T15:18:06,961 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/C of b423d13644c67f79070bddaad3c47156 into ef6cd149fdd54c75a773529791fedbee(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:06,961 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:06,961 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/C, priority=13, startTime=1731251886496; duration=0sec 2024-11-10T15:18:06,961 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:06,961 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:C 2024-11-10T15:18:06,989 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/5f19e9e30303400fabbce00d3ae082e8 2024-11-10T15:18:06,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/dc6ad878e2ac4159a7c9a5cbe97c6753 is 50, key is test_row_0/B:col10/1731251885536/Put/seqid=0 2024-11-10T15:18:07,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742245_1421 (size=12151) 2024-11-10T15:18:07,009 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/dc6ad878e2ac4159a7c9a5cbe97c6753 2024-11-10T15:18:07,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/1f04e2fed8cb4b0da16f0bd7731b7743 is 50, key is test_row_0/C:col10/1731251885536/Put/seqid=0 2024-11-10T15:18:07,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742246_1422 (size=12151) 2024-11-10T15:18:07,042 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/1f04e2fed8cb4b0da16f0bd7731b7743 2024-11-10T15:18:07,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/5f19e9e30303400fabbce00d3ae082e8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/5f19e9e30303400fabbce00d3ae082e8 2024-11-10T15:18:07,053 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/5f19e9e30303400fabbce00d3ae082e8, entries=150, sequenceid=222, filesize=30.4 K 2024-11-10T15:18:07,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/dc6ad878e2ac4159a7c9a5cbe97c6753 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/dc6ad878e2ac4159a7c9a5cbe97c6753 2024-11-10T15:18:07,059 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/dc6ad878e2ac4159a7c9a5cbe97c6753, entries=150, sequenceid=222, filesize=11.9 K 2024-11-10T15:18:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/1f04e2fed8cb4b0da16f0bd7731b7743 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/1f04e2fed8cb4b0da16f0bd7731b7743 2024-11-10T15:18:07,063 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/1f04e2fed8cb4b0da16f0bd7731b7743, entries=150, sequenceid=222, filesize=11.9 K 2024-11-10T15:18:07,064 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for b423d13644c67f79070bddaad3c47156 in 527ms, sequenceid=222, compaction requested=false 2024-11-10T15:18:07,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:07,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:07,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-10T15:18:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-10T15:18:07,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-10T15:18:07,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6210 sec 2024-11-10T15:18:07,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.6270 sec 2024-11-10T15:18:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:07,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T15:18:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:18:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:18:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:18:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:07,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104fd654970c4047b29ca68ebfdff405a1_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251887131/Put/seqid=0 2024-11-10T15:18:07,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742247_1423 (size=14794) 2024-11-10T15:18:07,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251947155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251947156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251947157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251947171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251947172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251947282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251947282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251947282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251947291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251947301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,342 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/2d495234b5aa44289364835ace87d7cf as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/2d495234b5aa44289364835ace87d7cf 2024-11-10T15:18:07,345 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/A of b423d13644c67f79070bddaad3c47156 into 2d495234b5aa44289364835ace87d7cf(size=30.9 K), total size for store is 61.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:07,346 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:07,346 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/A, priority=13, startTime=1731251886496; duration=0sec 2024-11-10T15:18:07,346 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:07,346 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:A 2024-11-10T15:18:07,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251947501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251947501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251947501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251947501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251947521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,554 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:07,560 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104fd654970c4047b29ca68ebfdff405a1_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104fd654970c4047b29ca68ebfdff405a1_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:07,561 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/40e51059626641dd9c4cc96b95996d37, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:07,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/40e51059626641dd9c4cc96b95996d37 is 175, key is test_row_0/A:col10/1731251887131/Put/seqid=0 2024-11-10T15:18:07,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742248_1424 (size=39749) 2024-11-10T15:18:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T15:18:07,592 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:07,592 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-10T15:18:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T15:18:07,595 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:07,596 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:07,596 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T15:18:07,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T15:18:07,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:07,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:07,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:07,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:07,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:07,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251947812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251947812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251947812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251947812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251947844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:07,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T15:18:07,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:07,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:07,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:07,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:07,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:07,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:07,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T15:18:07,968 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/40e51059626641dd9c4cc96b95996d37 2024-11-10T15:18:08,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/687eb13f7a854a68ac747c7998c6d4a4 is 50, key is test_row_0/B:col10/1731251887131/Put/seqid=0 2024-11-10T15:18:08,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742249_1425 (size=12151) 2024-11-10T15:18:08,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/687eb13f7a854a68ac747c7998c6d4a4 2024-11-10T15:18:08,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T15:18:08,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:08,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:08,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:08,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:08,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:08,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:08,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/8a6163b924294778a8ec6aff2446c13b is 50, key is test_row_0/C:col10/1731251887131/Put/seqid=0 2024-11-10T15:18:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T15:18:08,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742250_1426 (size=12151) 2024-11-10T15:18:08,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T15:18:08,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:08,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:08,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:08,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:08,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:08,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251948331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:08,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251948331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:08,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251948331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:08,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251948331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:08,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251948361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:08,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:08,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:08,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:08,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T15:18:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:08,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:08,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:08,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T15:18:08,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:08,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:08,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:08,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:08,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:08,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/8a6163b924294778a8ec6aff2446c13b 2024-11-10T15:18:08,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/40e51059626641dd9c4cc96b95996d37 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/40e51059626641dd9c4cc96b95996d37 2024-11-10T15:18:08,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/40e51059626641dd9c4cc96b95996d37, entries=200, sequenceid=239, filesize=38.8 K 2024-11-10T15:18:08,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/687eb13f7a854a68ac747c7998c6d4a4 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/687eb13f7a854a68ac747c7998c6d4a4 2024-11-10T15:18:08,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/687eb13f7a854a68ac747c7998c6d4a4, entries=150, sequenceid=239, filesize=11.9 K 2024-11-10T15:18:08,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/8a6163b924294778a8ec6aff2446c13b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/8a6163b924294778a8ec6aff2446c13b 2024-11-10T15:18:08,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/8a6163b924294778a8ec6aff2446c13b, entries=150, sequenceid=239, filesize=11.9 K 2024-11-10T15:18:08,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b423d13644c67f79070bddaad3c47156 in 1544ms, sequenceid=239, compaction requested=true 2024-11-10T15:18:08,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:08,677 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:08,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:08,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:08,677 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:08,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:08,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:08,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:08,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:08,679 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:08,679 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/A is initiating minor compaction (all files) 2024-11-10T15:18:08,679 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/A in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:08,680 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/2d495234b5aa44289364835ace87d7cf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/5f19e9e30303400fabbce00d3ae082e8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/40e51059626641dd9c4cc96b95996d37] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=100.1 K 2024-11-10T15:18:08,680 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:08,680 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/2d495234b5aa44289364835ace87d7cf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/5f19e9e30303400fabbce00d3ae082e8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/40e51059626641dd9c4cc96b95996d37] 2024-11-10T15:18:08,680 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:08,680 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/B is initiating minor compaction (all files) 2024-11-10T15:18:08,680 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/B in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:08,680 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/6b3bc412d8a84fd79d6b815f6fd99de1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/dc6ad878e2ac4159a7c9a5cbe97c6753, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/687eb13f7a854a68ac747c7998c6d4a4] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=36.1 K 2024-11-10T15:18:08,680 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d495234b5aa44289364835ace87d7cf, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731251884271 2024-11-10T15:18:08,681 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b3bc412d8a84fd79d6b815f6fd99de1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731251884271 2024-11-10T15:18:08,681 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f19e9e30303400fabbce00d3ae082e8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731251885527 2024-11-10T15:18:08,681 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40e51059626641dd9c4cc96b95996d37, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731251886735 2024-11-10T15:18:08,681 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting dc6ad878e2ac4159a7c9a5cbe97c6753, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731251885527 2024-11-10T15:18:08,681 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 687eb13f7a854a68ac747c7998c6d4a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731251886735 2024-11-10T15:18:08,689 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:08,690 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#B#compaction#352 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:08,691 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/ab2fe108ed664bfaa76db49293841f31 is 50, key is test_row_0/B:col10/1731251887131/Put/seqid=0 2024-11-10T15:18:08,701 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411107a8a67511f8441ebab58c7ec6a0d252f_b423d13644c67f79070bddaad3c47156 store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:08,703 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411107a8a67511f8441ebab58c7ec6a0d252f_b423d13644c67f79070bddaad3c47156, store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:08,703 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107a8a67511f8441ebab58c7ec6a0d252f_b423d13644c67f79070bddaad3c47156 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:08,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742251_1427 (size=12765) 2024-11-10T15:18:08,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742252_1428 (size=4469) 2024-11-10T15:18:08,711 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#A#compaction#351 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:08,711 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/53aea48b91cf41d69f5a618d47ca6257 is 175, key is test_row_0/A:col10/1731251887131/Put/seqid=0 2024-11-10T15:18:08,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742253_1429 (size=31719) 2024-11-10T15:18:08,720 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/53aea48b91cf41d69f5a618d47ca6257 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/53aea48b91cf41d69f5a618d47ca6257 2024-11-10T15:18:08,724 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/A of b423d13644c67f79070bddaad3c47156 into 53aea48b91cf41d69f5a618d47ca6257(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:08,724 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:08,724 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/A, priority=13, startTime=1731251888677; duration=0sec 2024-11-10T15:18:08,724 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:08,724 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:A 2024-11-10T15:18:08,724 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:08,725 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:08,725 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/C is initiating minor compaction (all files) 2024-11-10T15:18:08,725 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/C in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:08,725 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/ef6cd149fdd54c75a773529791fedbee, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/1f04e2fed8cb4b0da16f0bd7731b7743, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/8a6163b924294778a8ec6aff2446c13b] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=36.1 K 2024-11-10T15:18:08,726 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef6cd149fdd54c75a773529791fedbee, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731251884271 2024-11-10T15:18:08,726 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f04e2fed8cb4b0da16f0bd7731b7743, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731251885527 2024-11-10T15:18:08,726 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a6163b924294778a8ec6aff2446c13b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731251886735 2024-11-10T15:18:08,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T15:18:08,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:08,732 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T15:18:08,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:18:08,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:08,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:18:08,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:08,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:18:08,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:08,734 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#C#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:08,735 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/75d977f388f5402ca74dea2b258b06dc is 50, key is test_row_0/C:col10/1731251887131/Put/seqid=0 2024-11-10T15:18:08,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742254_1430 (size=12765) 2024-11-10T15:18:08,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ae1a4351a7474967b3a0908a12943c62_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251887156/Put/seqid=0 2024-11-10T15:18:08,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T15:18:08,759 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/75d977f388f5402ca74dea2b258b06dc as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/75d977f388f5402ca74dea2b258b06dc 2024-11-10T15:18:08,767 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/C of b423d13644c67f79070bddaad3c47156 into 75d977f388f5402ca74dea2b258b06dc(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:08,767 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:08,767 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/C, priority=13, startTime=1731251888677; duration=0sec 2024-11-10T15:18:08,767 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:08,767 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:C 2024-11-10T15:18:08,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742255_1431 (size=12304) 2024-11-10T15:18:09,127 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/ab2fe108ed664bfaa76db49293841f31 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/ab2fe108ed664bfaa76db49293841f31 2024-11-10T15:18:09,142 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/B of b423d13644c67f79070bddaad3c47156 into ab2fe108ed664bfaa76db49293841f31(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:09,142 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:09,142 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/B, priority=13, startTime=1731251888677; duration=0sec 2024-11-10T15:18:09,142 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:09,142 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:B 2024-11-10T15:18:09,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:09,175 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ae1a4351a7474967b3a0908a12943c62_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ae1a4351a7474967b3a0908a12943c62_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:09,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/2c66237c93b44533a17b984a7294ce2b, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/2c66237c93b44533a17b984a7294ce2b is 175, key is test_row_0/A:col10/1731251887156/Put/seqid=0 2024-11-10T15:18:09,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742256_1432 (size=31105) 2024-11-10T15:18:09,197 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/2c66237c93b44533a17b984a7294ce2b 2024-11-10T15:18:09,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/3f4b3d2609f44ead978b9135344c6509 is 50, key is test_row_0/B:col10/1731251887156/Put/seqid=0 2024-11-10T15:18:09,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742257_1433 (size=12151) 2024-11-10T15:18:09,238 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/3f4b3d2609f44ead978b9135344c6509 2024-11-10T15:18:09,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/f4c99602a9f4432f9288cc12f88f8c6a is 50, key is test_row_0/C:col10/1731251887156/Put/seqid=0 2024-11-10T15:18:09,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742258_1434 (size=12151) 2024-11-10T15:18:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:09,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:09,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251949381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251949381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251949389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,398 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in b423d13644c67f79070bddaad3c47156 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:18:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251949395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251949397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,425 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T15:18:09,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251949502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251949504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251949511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251949522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,657 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/f4c99602a9f4432f9288cc12f88f8c6a 2024-11-10T15:18:09,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/2c66237c93b44533a17b984a7294ce2b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/2c66237c93b44533a17b984a7294ce2b 2024-11-10T15:18:09,685 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/2c66237c93b44533a17b984a7294ce2b, entries=150, sequenceid=257, filesize=30.4 K 2024-11-10T15:18:09,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/3f4b3d2609f44ead978b9135344c6509 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/3f4b3d2609f44ead978b9135344c6509 2024-11-10T15:18:09,702 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/3f4b3d2609f44ead978b9135344c6509, entries=150, sequenceid=257, filesize=11.9 K 2024-11-10T15:18:09,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/f4c99602a9f4432f9288cc12f88f8c6a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/f4c99602a9f4432f9288cc12f88f8c6a 2024-11-10T15:18:09,720 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/f4c99602a9f4432f9288cc12f88f8c6a, entries=150, sequenceid=257, filesize=11.9 K 2024-11-10T15:18:09,722 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b423d13644c67f79070bddaad3c47156 in 990ms, sequenceid=257, compaction requested=false 2024-11-10T15:18:09,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:09,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:09,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-10T15:18:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-10T15:18:09,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T15:18:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:09,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:18:09,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:09,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:18:09,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:09,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:18:09,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:09,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-10T15:18:09,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1280 sec 2024-11-10T15:18:09,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.1340 sec 2024-11-10T15:18:09,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111057e14cc2dbdb4a078f830ed521dcec47_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251889395/Put/seqid=0 2024-11-10T15:18:09,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742259_1435 (size=14994) 2024-11-10T15:18:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T15:18:09,762 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:09,763 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-10T15:18:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T15:18:09,765 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:09,766 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:09,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:09,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251949769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251949770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251949771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251949772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T15:18:09,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251949891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251949891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251949891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251949891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:09,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T15:18:09,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:09,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:09,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:09,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:09,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:09,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T15:18:10,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:10,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:10,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:10,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T15:18:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251950101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251950101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251950101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:10,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251950101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:10,142 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:10,146 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111057e14cc2dbdb4a078f830ed521dcec47_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111057e14cc2dbdb4a078f830ed521dcec47_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:10,149 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/188c38f0d3c44f3b83633081469ccf5a, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:10,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/188c38f0d3c44f3b83633081469ccf5a is 175, key is test_row_0/A:col10/1731251889395/Put/seqid=0 2024-11-10T15:18:10,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742260_1436 (size=39949) 2024-11-10T15:18:10,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T15:18:10,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:10,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:10,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:10,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T15:18:10,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:10,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:10,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:10,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T15:18:10,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251950411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:10,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251950421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251950421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251950422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:10,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T15:18:10,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:10,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:10,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:10,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,556 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/188c38f0d3c44f3b83633081469ccf5a 2024-11-10T15:18:10,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/a6879d4dfed54710a64f5c7e077a2f70 is 50, key is test_row_0/B:col10/1731251889395/Put/seqid=0 2024-11-10T15:18:10,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742261_1437 (size=12301) 2024-11-10T15:18:10,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T15:18:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:10,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T15:18:10,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:10,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:10,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:10,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T15:18:10,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251950932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:10,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251950932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:10,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251950932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:10,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251950932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:10,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:10,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:10,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/a6879d4dfed54710a64f5c7e077a2f70 2024-11-10T15:18:10,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T15:18:10,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:10,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:10,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:10,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:10,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:11,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/d4e80bb17fce4760b42869d41a7cff0d is 50, key is test_row_0/C:col10/1731251889395/Put/seqid=0 2024-11-10T15:18:11,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742262_1438 (size=12301) 2024-11-10T15:18:11,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T15:18:11,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:11,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:11,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:11,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:11,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:11,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T15:18:11,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:11,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:11,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:11,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:11,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:11,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/d4e80bb17fce4760b42869d41a7cff0d 2024-11-10T15:18:11,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/188c38f0d3c44f3b83633081469ccf5a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/188c38f0d3c44f3b83633081469ccf5a 2024-11-10T15:18:11,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45842 deadline: 1731251951421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:11,431 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in b423d13644c67f79070bddaad3c47156 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:18:11,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:11,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:11,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:11,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/188c38f0d3c44f3b83633081469ccf5a, entries=200, sequenceid=276, filesize=39.0 K 2024-11-10T15:18:11,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/a6879d4dfed54710a64f5c7e077a2f70 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/a6879d4dfed54710a64f5c7e077a2f70 2024-11-10T15:18:11,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/a6879d4dfed54710a64f5c7e077a2f70, entries=150, sequenceid=276, filesize=12.0 K 2024-11-10T15:18:11,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/d4e80bb17fce4760b42869d41a7cff0d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/d4e80bb17fce4760b42869d41a7cff0d 2024-11-10T15:18:11,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/d4e80bb17fce4760b42869d41a7cff0d, entries=150, sequenceid=276, filesize=12.0 K 2024-11-10T15:18:11,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for b423d13644c67f79070bddaad3c47156 in 1724ms, sequenceid=276, compaction requested=true 2024-11-10T15:18:11,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:11,448 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:11,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:11,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:11,448 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:11,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:11,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:11,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:11,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:11,449 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:11,449 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/A is initiating minor compaction (all files) 2024-11-10T15:18:11,449 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/A in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:11,449 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/53aea48b91cf41d69f5a618d47ca6257, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/2c66237c93b44533a17b984a7294ce2b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/188c38f0d3c44f3b83633081469ccf5a] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=100.4 K 2024-11-10T15:18:11,450 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:11,450 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/53aea48b91cf41d69f5a618d47ca6257, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/2c66237c93b44533a17b984a7294ce2b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/188c38f0d3c44f3b83633081469ccf5a] 2024-11-10T15:18:11,450 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:11,450 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/B is initiating minor compaction (all files) 2024-11-10T15:18:11,450 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/B in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:11,450 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53aea48b91cf41d69f5a618d47ca6257, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731251886735 2024-11-10T15:18:11,450 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/ab2fe108ed664bfaa76db49293841f31, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/3f4b3d2609f44ead978b9135344c6509, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/a6879d4dfed54710a64f5c7e077a2f70] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=36.3 K 2024-11-10T15:18:11,450 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c66237c93b44533a17b984a7294ce2b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731251887155 2024-11-10T15:18:11,450 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting ab2fe108ed664bfaa76db49293841f31, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731251886735 2024-11-10T15:18:11,451 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 188c38f0d3c44f3b83633081469ccf5a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731251889378 2024-11-10T15:18:11,451 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f4b3d2609f44ead978b9135344c6509, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731251887155 2024-11-10T15:18:11,452 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting a6879d4dfed54710a64f5c7e077a2f70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731251889393 2024-11-10T15:18:11,458 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:11,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T15:18:11,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:11,459 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T15:18:11,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:18:11,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:11,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:18:11,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:11,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:18:11,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:11,460 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100423080d49da4cea93c6f60781271be0_b423d13644c67f79070bddaad3c47156 store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:11,460 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#B#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:11,461 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/21133037b5824b7d817e175a44ec7ab3 is 50, key is test_row_0/B:col10/1731251889395/Put/seqid=0 2024-11-10T15:18:11,462 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100423080d49da4cea93c6f60781271be0_b423d13644c67f79070bddaad3c47156, store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:11,462 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100423080d49da4cea93c6f60781271be0_b423d13644c67f79070bddaad3c47156 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:11,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111014d2868335754603af8eda740b2cb97f_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251889766/Put/seqid=0 2024-11-10T15:18:11,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742264_1440 (size=13017) 2024-11-10T15:18:11,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742263_1439 (size=4469) 2024-11-10T15:18:11,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742265_1441 (size=12454) 2024-11-10T15:18:11,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:11,492 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111014d2868335754603af8eda740b2cb97f_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111014d2868335754603af8eda740b2cb97f_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:11,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/8d926be9b32e4d3e8ea7b4e7010ab5ff, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:11,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/8d926be9b32e4d3e8ea7b4e7010ab5ff is 175, key is test_row_0/A:col10/1731251889766/Put/seqid=0 2024-11-10T15:18:11,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742266_1442 (size=31255) 2024-11-10T15:18:11,520 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/8d926be9b32e4d3e8ea7b4e7010ab5ff 2024-11-10T15:18:11,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/fd5ac80db09e455eb3a8659a134f6d06 is 50, key is test_row_0/B:col10/1731251889766/Put/seqid=0 2024-11-10T15:18:11,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742267_1443 (size=12301) 2024-11-10T15:18:11,876 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/21133037b5824b7d817e175a44ec7ab3 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/21133037b5824b7d817e175a44ec7ab3 2024-11-10T15:18:11,877 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#A#compaction#360 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:11,878 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/41a95488893d48ce8ebbc4a47094887b is 175, key is test_row_0/A:col10/1731251889395/Put/seqid=0 2024-11-10T15:18:11,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742268_1444 (size=31971) 2024-11-10T15:18:11,884 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/B of b423d13644c67f79070bddaad3c47156 into 21133037b5824b7d817e175a44ec7ab3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:11,884 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:11,884 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/B, priority=13, startTime=1731251891448; duration=0sec 2024-11-10T15:18:11,884 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:11,884 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:B 2024-11-10T15:18:11,885 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:11,885 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:11,885 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/C is initiating minor compaction (all files) 2024-11-10T15:18:11,886 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/C in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:11,886 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/75d977f388f5402ca74dea2b258b06dc, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/f4c99602a9f4432f9288cc12f88f8c6a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/d4e80bb17fce4760b42869d41a7cff0d] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=36.3 K 2024-11-10T15:18:11,886 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 75d977f388f5402ca74dea2b258b06dc, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731251886735 2024-11-10T15:18:11,889 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting f4c99602a9f4432f9288cc12f88f8c6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731251887155 2024-11-10T15:18:11,890 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting d4e80bb17fce4760b42869d41a7cff0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731251889393 2024-11-10T15:18:11,891 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/41a95488893d48ce8ebbc4a47094887b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/41a95488893d48ce8ebbc4a47094887b 2024-11-10T15:18:11,902 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/A of b423d13644c67f79070bddaad3c47156 into 41a95488893d48ce8ebbc4a47094887b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:11,902 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:11,902 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/A, priority=13, startTime=1731251891447; duration=0sec 2024-11-10T15:18:11,903 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:11,903 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:A 2024-11-10T15:18:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T15:18:11,929 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#C#compaction#364 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:11,929 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/be63bdc3089f424085757c2c42fb926d is 50, key is test_row_0/C:col10/1731251889395/Put/seqid=0 2024-11-10T15:18:11,942 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/fd5ac80db09e455eb3a8659a134f6d06 2024-11-10T15:18:11,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742269_1445 (size=13017) 2024-11-10T15:18:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. as already flushing 2024-11-10T15:18:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:11,964 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/be63bdc3089f424085757c2c42fb926d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/be63bdc3089f424085757c2c42fb926d 2024-11-10T15:18:11,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/5e8b4bc2904c4d92a63670fc309d0568 is 50, key is test_row_0/C:col10/1731251889766/Put/seqid=0 2024-11-10T15:18:11,970 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/C of b423d13644c67f79070bddaad3c47156 into be63bdc3089f424085757c2c42fb926d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:11,970 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:11,970 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/C, priority=13, startTime=1731251891448; duration=0sec 2024-11-10T15:18:11,970 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:11,970 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:C 2024-11-10T15:18:11,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742270_1446 (size=12301) 2024-11-10T15:18:11,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251951980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:11,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251951990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:11,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251951990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251951992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251952101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251952102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251952102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251952111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251952322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251952322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251952323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251952332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,378 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/5e8b4bc2904c4d92a63670fc309d0568 2024-11-10T15:18:12,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/8d926be9b32e4d3e8ea7b4e7010ab5ff as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/8d926be9b32e4d3e8ea7b4e7010ab5ff 2024-11-10T15:18:12,387 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/8d926be9b32e4d3e8ea7b4e7010ab5ff, entries=150, sequenceid=291, filesize=30.5 K 2024-11-10T15:18:12,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/fd5ac80db09e455eb3a8659a134f6d06 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/fd5ac80db09e455eb3a8659a134f6d06 2024-11-10T15:18:12,391 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/fd5ac80db09e455eb3a8659a134f6d06, entries=150, sequenceid=291, filesize=12.0 K 2024-11-10T15:18:12,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/5e8b4bc2904c4d92a63670fc309d0568 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/5e8b4bc2904c4d92a63670fc309d0568 2024-11-10T15:18:12,402 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/5e8b4bc2904c4d92a63670fc309d0568, entries=150, sequenceid=291, filesize=12.0 K 2024-11-10T15:18:12,403 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b423d13644c67f79070bddaad3c47156 in 944ms, sequenceid=291, compaction requested=false 2024-11-10T15:18:12,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:12,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:12,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-10T15:18:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-10T15:18:12,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-10T15:18:12,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6380 sec 2024-11-10T15:18:12,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.6440 sec 2024-11-10T15:18:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:12,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T15:18:12,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:18:12,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:12,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:18:12,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:12,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:18:12,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:12,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110149ca832560340f8b49dd850c0a06b75_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251891990/Put/seqid=0 2024-11-10T15:18:12,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742271_1447 (size=17534) 2024-11-10T15:18:12,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251952690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251952705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251952707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251952708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,728 INFO [Thread-1625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1625. 2024-11-10T15:18:12,728 DEBUG [Thread-1625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:12,728 DEBUG [Thread-1625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:12,728 DEBUG [Thread-1625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:12,728 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:12,728 INFO [Thread-1621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1621. 2024-11-10T15:18:12,729 DEBUG [Thread-1621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:12,729 DEBUG [Thread-1621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:12,729 DEBUG [Thread-1621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:12,729 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:12,731 INFO [Thread-1627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1627. 2024-11-10T15:18:12,731 DEBUG [Thread-1627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:12,731 DEBUG [Thread-1627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:12,731 DEBUG [Thread-1627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:12,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:12,743 INFO [Thread-1619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1619. 2024-11-10T15:18:12,743 DEBUG [Thread-1619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:12,743 DEBUG [Thread-1619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:12,744 DEBUG [Thread-1619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:12,744 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:12,744 INFO [Thread-1623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1623. 2024-11-10T15:18:12,744 DEBUG [Thread-1623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:12,744 DEBUG [Thread-1623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:12,744 DEBUG [Thread-1623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:12,744 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:12,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251952812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251952822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251952831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251952831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:13,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:13,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251953022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:13,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:13,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251953032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:13,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:13,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:13,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251953041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:13,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251953041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:13,076 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:13,080 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110149ca832560340f8b49dd850c0a06b75_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110149ca832560340f8b49dd850c0a06b75_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:13,081 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/6d891f6e75174aa5a5e81ae4f2677c10, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:13,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/6d891f6e75174aa5a5e81ae4f2677c10 is 175, key is test_row_0/A:col10/1731251891990/Put/seqid=0 2024-11-10T15:18:13,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742272_1448 (size=48639) 2024-11-10T15:18:13,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251953331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:13,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251953342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:13,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:13,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251953352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251953352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:13,485 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/6d891f6e75174aa5a5e81ae4f2677c10 2024-11-10T15:18:13,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/65313c0bedb24f85b54e979ea739ae60 is 50, key is test_row_0/B:col10/1731251891990/Put/seqid=0 2024-11-10T15:18:13,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742273_1449 (size=12301) 2024-11-10T15:18:13,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/65313c0bedb24f85b54e979ea739ae60 2024-11-10T15:18:13,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/aa2584595ee94e3cb0a6f9b467badc1d is 50, key is test_row_0/C:col10/1731251891990/Put/seqid=0 2024-11-10T15:18:13,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742274_1450 (size=12301) 2024-11-10T15:18:13,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:13,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45818 deadline: 1731251953843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:13,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:13,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45848 deadline: 1731251953852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:13,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:13,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:13,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45794 deadline: 1731251953861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45768 deadline: 1731251953861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:13,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T15:18:13,931 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:13,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/aa2584595ee94e3cb0a6f9b467badc1d 2024-11-10T15:18:13,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/6d891f6e75174aa5a5e81ae4f2677c10 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/6d891f6e75174aa5a5e81ae4f2677c10 2024-11-10T15:18:13,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/6d891f6e75174aa5a5e81ae4f2677c10, entries=250, sequenceid=313, filesize=47.5 K 2024-11-10T15:18:13,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/65313c0bedb24f85b54e979ea739ae60 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/65313c0bedb24f85b54e979ea739ae60 2024-11-10T15:18:13,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/65313c0bedb24f85b54e979ea739ae60, entries=150, sequenceid=313, filesize=12.0 K 2024-11-10T15:18:13,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/aa2584595ee94e3cb0a6f9b467badc1d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/aa2584595ee94e3cb0a6f9b467badc1d 2024-11-10T15:18:13,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/aa2584595ee94e3cb0a6f9b467badc1d, entries=150, sequenceid=313, filesize=12.0 K 2024-11-10T15:18:13,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b423d13644c67f79070bddaad3c47156 in 1300ms, sequenceid=313, compaction requested=true 2024-11-10T15:18:13,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:13,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:13,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:13,949 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:13,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:13,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:13,949 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:13,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b423d13644c67f79070bddaad3c47156:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:13,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:13,950 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:13,950 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:13,950 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/B is initiating minor compaction (all files) 2024-11-10T15:18:13,950 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/A is initiating minor compaction (all files) 2024-11-10T15:18:13,950 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/A in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:13,950 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/B in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:13,950 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/21133037b5824b7d817e175a44ec7ab3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/fd5ac80db09e455eb3a8659a134f6d06, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/65313c0bedb24f85b54e979ea739ae60] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=36.7 K 2024-11-10T15:18:13,950 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/41a95488893d48ce8ebbc4a47094887b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/8d926be9b32e4d3e8ea7b4e7010ab5ff, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/6d891f6e75174aa5a5e81ae4f2677c10] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=109.2 K 2024-11-10T15:18:13,950 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:13,950 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/41a95488893d48ce8ebbc4a47094887b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/8d926be9b32e4d3e8ea7b4e7010ab5ff, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/6d891f6e75174aa5a5e81ae4f2677c10] 2024-11-10T15:18:13,950 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 21133037b5824b7d817e175a44ec7ab3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731251889393 2024-11-10T15:18:13,950 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41a95488893d48ce8ebbc4a47094887b, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731251889393 2024-11-10T15:18:13,950 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting fd5ac80db09e455eb3a8659a134f6d06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731251889766 2024-11-10T15:18:13,950 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d926be9b32e4d3e8ea7b4e7010ab5ff, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731251889766 2024-11-10T15:18:13,951 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 65313c0bedb24f85b54e979ea739ae60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731251891974 2024-11-10T15:18:13,951 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d891f6e75174aa5a5e81ae4f2677c10, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731251891974 2024-11-10T15:18:13,956 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:13,956 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/48b09b110e0642afb39c028b6edfbf26 is 50, key is test_row_0/B:col10/1731251891990/Put/seqid=0 2024-11-10T15:18:13,958 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:13,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742275_1451 (size=13119) 2024-11-10T15:18:13,961 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110a802f6c673f24231acca52f11ea10f26_b423d13644c67f79070bddaad3c47156 store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:13,965 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/48b09b110e0642afb39c028b6edfbf26 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/48b09b110e0642afb39c028b6edfbf26 2024-11-10T15:18:13,969 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/B of b423d13644c67f79070bddaad3c47156 into 48b09b110e0642afb39c028b6edfbf26(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:13,969 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:13,969 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/B, priority=13, startTime=1731251893949; duration=0sec 2024-11-10T15:18:13,970 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:13,970 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:B 2024-11-10T15:18:13,970 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:13,971 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:13,971 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): b423d13644c67f79070bddaad3c47156/C is initiating minor compaction (all files) 2024-11-10T15:18:13,971 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b423d13644c67f79070bddaad3c47156/C in TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:13,971 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/be63bdc3089f424085757c2c42fb926d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/5e8b4bc2904c4d92a63670fc309d0568, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/aa2584595ee94e3cb0a6f9b467badc1d] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp, totalSize=36.7 K 2024-11-10T15:18:13,971 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting be63bdc3089f424085757c2c42fb926d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731251889393 2024-11-10T15:18:13,971 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e8b4bc2904c4d92a63670fc309d0568, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731251889766 2024-11-10T15:18:13,972 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting aa2584595ee94e3cb0a6f9b467badc1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731251891974 2024-11-10T15:18:13,974 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110a802f6c673f24231acca52f11ea10f26_b423d13644c67f79070bddaad3c47156, store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:13,974 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a802f6c673f24231acca52f11ea10f26_b423d13644c67f79070bddaad3c47156 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:13,980 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#C#compaction#371 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:13,980 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/ee9a8297d9b14ee595e92779b66732b7 is 50, key is test_row_0/C:col10/1731251891990/Put/seqid=0 2024-11-10T15:18:13,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742276_1452 (size=4469) 2024-11-10T15:18:13,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742277_1453 (size=13119) 2024-11-10T15:18:14,384 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b423d13644c67f79070bddaad3c47156#A#compaction#370 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:14,385 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/ff76343e3f614f808925900859900801 is 175, key is test_row_0/A:col10/1731251891990/Put/seqid=0 2024-11-10T15:18:14,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742278_1454 (size=32073) 2024-11-10T15:18:14,402 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/ee9a8297d9b14ee595e92779b66732b7 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/ee9a8297d9b14ee595e92779b66732b7 2024-11-10T15:18:14,407 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/C of b423d13644c67f79070bddaad3c47156 into ee9a8297d9b14ee595e92779b66732b7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:14,407 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:14,407 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/C, priority=13, startTime=1731251893949; duration=0sec 2024-11-10T15:18:14,408 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:14,408 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:C 2024-11-10T15:18:14,539 WARN [regionserver/c0771061be61:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-10T15:18:14,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/a7b9ddb34c674c50bb8a8794e3ffbd13, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/e284de4042414398a3a708247559ad96, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/5269132e2ad247ec8a361d3aa47155ef, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/f769af09fa5d410a9bf2dca2ec9dd95b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/01b2fb8995a94e3a8880981fb835c6a5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/dcec43efe0ef4808bad146ef82ee5033, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/469d4056b7a04949af886e0f1df0f5f6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/9516988daecb448284063fbd0208804b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/38609d6211bd4580a0e4155ccfb201e8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/dfc6ab5c3e3c45718530d2866f117522, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/0ca68244fc234418bacb11e8ff4ef0ce, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/c3363eacf3f244bab8ab4b4d8c89a666, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/1adce74f432745ff98cb76e4d7ae3f82, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/000098616c4f4a57b2a8cabcb94361e0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/38ce898e80e64e1c89fe528a5b0eae53, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/2d495234b5aa44289364835ace87d7cf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/5f19e9e30303400fabbce00d3ae082e8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/40e51059626641dd9c4cc96b95996d37, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/53aea48b91cf41d69f5a618d47ca6257, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/2c66237c93b44533a17b984a7294ce2b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/188c38f0d3c44f3b83633081469ccf5a] to archive 2024-11-10T15:18:14,543 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:18:14,545 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/a7b9ddb34c674c50bb8a8794e3ffbd13 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/a7b9ddb34c674c50bb8a8794e3ffbd13 2024-11-10T15:18:14,546 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/e284de4042414398a3a708247559ad96 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/e284de4042414398a3a708247559ad96 2024-11-10T15:18:14,547 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/5269132e2ad247ec8a361d3aa47155ef to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/5269132e2ad247ec8a361d3aa47155ef 2024-11-10T15:18:14,548 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/f769af09fa5d410a9bf2dca2ec9dd95b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/f769af09fa5d410a9bf2dca2ec9dd95b 2024-11-10T15:18:14,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/01b2fb8995a94e3a8880981fb835c6a5 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/01b2fb8995a94e3a8880981fb835c6a5 2024-11-10T15:18:14,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/dcec43efe0ef4808bad146ef82ee5033 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/dcec43efe0ef4808bad146ef82ee5033 2024-11-10T15:18:14,550 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/469d4056b7a04949af886e0f1df0f5f6 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/469d4056b7a04949af886e0f1df0f5f6 2024-11-10T15:18:14,551 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/9516988daecb448284063fbd0208804b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/9516988daecb448284063fbd0208804b 2024-11-10T15:18:14,552 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/38609d6211bd4580a0e4155ccfb201e8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/38609d6211bd4580a0e4155ccfb201e8 2024-11-10T15:18:14,553 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/dfc6ab5c3e3c45718530d2866f117522 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/dfc6ab5c3e3c45718530d2866f117522 2024-11-10T15:18:14,554 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/0ca68244fc234418bacb11e8ff4ef0ce to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/0ca68244fc234418bacb11e8ff4ef0ce 2024-11-10T15:18:14,554 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/c3363eacf3f244bab8ab4b4d8c89a666 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/c3363eacf3f244bab8ab4b4d8c89a666 2024-11-10T15:18:14,555 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/1adce74f432745ff98cb76e4d7ae3f82 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/1adce74f432745ff98cb76e4d7ae3f82 2024-11-10T15:18:14,556 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/000098616c4f4a57b2a8cabcb94361e0 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/000098616c4f4a57b2a8cabcb94361e0 2024-11-10T15:18:14,558 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/38ce898e80e64e1c89fe528a5b0eae53 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/38ce898e80e64e1c89fe528a5b0eae53 2024-11-10T15:18:14,559 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/2d495234b5aa44289364835ace87d7cf to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/2d495234b5aa44289364835ace87d7cf 2024-11-10T15:18:14,561 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/5f19e9e30303400fabbce00d3ae082e8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/5f19e9e30303400fabbce00d3ae082e8 2024-11-10T15:18:14,562 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/40e51059626641dd9c4cc96b95996d37 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/40e51059626641dd9c4cc96b95996d37 2024-11-10T15:18:14,563 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/53aea48b91cf41d69f5a618d47ca6257 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/53aea48b91cf41d69f5a618d47ca6257 2024-11-10T15:18:14,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/2c66237c93b44533a17b984a7294ce2b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/2c66237c93b44533a17b984a7294ce2b 2024-11-10T15:18:14,565 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/188c38f0d3c44f3b83633081469ccf5a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/188c38f0d3c44f3b83633081469ccf5a 2024-11-10T15:18:14,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/5a38518941ee48fda2bbd2ad29c25ccc, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/706712ca6bbc47858f8b87de9d7bf4f1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/cd64bc77c22547749b306673c525929e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/f4fbc27a340e479c9697c03843d3936f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/01485841a82e4bf2bc542b5e975a6da2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/45f22cfc2cf44859a40470c82a0b7415, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/1ac0e648ec3d4259b8d05dc929be4240, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/0b3169639973491384265dce92c23e0b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/262618b24cfc419abf4d56d430d5d26d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/b42585cb30b547079ae25ce758b50c37, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/aca8a2a3789b4eb5bd51f6c5c1d5e2da, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/024459843af8418babc2889715f43585, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/0e39a49eb5da43b0811fe9a8e3ce6f9a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/85908f605e944d25873be4238aedbf07, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/6b3bc412d8a84fd79d6b815f6fd99de1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/11ea648d288c4329b17d314b8192e353, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/dc6ad878e2ac4159a7c9a5cbe97c6753, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/ab2fe108ed664bfaa76db49293841f31, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/687eb13f7a854a68ac747c7998c6d4a4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/3f4b3d2609f44ead978b9135344c6509, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/21133037b5824b7d817e175a44ec7ab3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/a6879d4dfed54710a64f5c7e077a2f70, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/fd5ac80db09e455eb3a8659a134f6d06, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/65313c0bedb24f85b54e979ea739ae60] to archive 2024-11-10T15:18:14,576 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:18:14,579 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/5a38518941ee48fda2bbd2ad29c25ccc to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/5a38518941ee48fda2bbd2ad29c25ccc 2024-11-10T15:18:14,580 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/706712ca6bbc47858f8b87de9d7bf4f1 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/706712ca6bbc47858f8b87de9d7bf4f1 2024-11-10T15:18:14,581 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/cd64bc77c22547749b306673c525929e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/cd64bc77c22547749b306673c525929e 2024-11-10T15:18:14,588 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/f4fbc27a340e479c9697c03843d3936f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/f4fbc27a340e479c9697c03843d3936f 2024-11-10T15:18:14,590 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/01485841a82e4bf2bc542b5e975a6da2 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/01485841a82e4bf2bc542b5e975a6da2 2024-11-10T15:18:14,591 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/45f22cfc2cf44859a40470c82a0b7415 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/45f22cfc2cf44859a40470c82a0b7415 2024-11-10T15:18:14,592 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/1ac0e648ec3d4259b8d05dc929be4240 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/1ac0e648ec3d4259b8d05dc929be4240 2024-11-10T15:18:14,593 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/0b3169639973491384265dce92c23e0b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/0b3169639973491384265dce92c23e0b 2024-11-10T15:18:14,594 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/262618b24cfc419abf4d56d430d5d26d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/262618b24cfc419abf4d56d430d5d26d 2024-11-10T15:18:14,595 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/b42585cb30b547079ae25ce758b50c37 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/b42585cb30b547079ae25ce758b50c37 2024-11-10T15:18:14,595 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/aca8a2a3789b4eb5bd51f6c5c1d5e2da to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/aca8a2a3789b4eb5bd51f6c5c1d5e2da 2024-11-10T15:18:14,596 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/024459843af8418babc2889715f43585 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/024459843af8418babc2889715f43585 2024-11-10T15:18:14,597 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/0e39a49eb5da43b0811fe9a8e3ce6f9a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/0e39a49eb5da43b0811fe9a8e3ce6f9a 2024-11-10T15:18:14,598 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/85908f605e944d25873be4238aedbf07 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/85908f605e944d25873be4238aedbf07 2024-11-10T15:18:14,599 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/6b3bc412d8a84fd79d6b815f6fd99de1 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/6b3bc412d8a84fd79d6b815f6fd99de1 2024-11-10T15:18:14,599 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/11ea648d288c4329b17d314b8192e353 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/11ea648d288c4329b17d314b8192e353 2024-11-10T15:18:14,600 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/dc6ad878e2ac4159a7c9a5cbe97c6753 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/dc6ad878e2ac4159a7c9a5cbe97c6753 2024-11-10T15:18:14,601 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/ab2fe108ed664bfaa76db49293841f31 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/ab2fe108ed664bfaa76db49293841f31 2024-11-10T15:18:14,602 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/687eb13f7a854a68ac747c7998c6d4a4 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/687eb13f7a854a68ac747c7998c6d4a4 2024-11-10T15:18:14,603 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/3f4b3d2609f44ead978b9135344c6509 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/3f4b3d2609f44ead978b9135344c6509 2024-11-10T15:18:14,603 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/21133037b5824b7d817e175a44ec7ab3 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/21133037b5824b7d817e175a44ec7ab3 2024-11-10T15:18:14,604 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/a6879d4dfed54710a64f5c7e077a2f70 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/a6879d4dfed54710a64f5c7e077a2f70 2024-11-10T15:18:14,605 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/fd5ac80db09e455eb3a8659a134f6d06 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/fd5ac80db09e455eb3a8659a134f6d06 2024-11-10T15:18:14,606 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/65313c0bedb24f85b54e979ea739ae60 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/65313c0bedb24f85b54e979ea739ae60 2024-11-10T15:18:14,608 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/9428dd32d4c6438b886a297eb4615e79, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/682748a4c2c9406992bcb2bd16f7589e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/04a48bb966594780afbae3e058c933a3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/6975eb0c7ba64c638696a65811d5ac61, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/cf406ba3b20749429dff4285d96e8504, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/28d67a32db844c7a9220c1d82468fb35, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/99374be1314c4deba6959214d0bc6554, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/1d46dae6a32a46c3b5ed3d292e9038bf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/4e4eadd0971d4517a2b750a9742b21f8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/46260ac2490d43029ef4248a3ab6708d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/ec1b4b410b95413e87a296e5ce4d3d7a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/854f9c49e418416e95aec5ad03e48d0b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/9257842fa5a64ccd977be9194c719b46, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/b988868064c44ce28e8c149fdd6b89d6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/ef6cd149fdd54c75a773529791fedbee, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/b839c9b13a904f329be2dce822830938, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/1f04e2fed8cb4b0da16f0bd7731b7743, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/75d977f388f5402ca74dea2b258b06dc, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/8a6163b924294778a8ec6aff2446c13b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/f4c99602a9f4432f9288cc12f88f8c6a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/be63bdc3089f424085757c2c42fb926d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/d4e80bb17fce4760b42869d41a7cff0d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/5e8b4bc2904c4d92a63670fc309d0568, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/aa2584595ee94e3cb0a6f9b467badc1d] to archive 2024-11-10T15:18:14,609 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:18:14,610 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/9428dd32d4c6438b886a297eb4615e79 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/9428dd32d4c6438b886a297eb4615e79 2024-11-10T15:18:14,611 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/682748a4c2c9406992bcb2bd16f7589e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/682748a4c2c9406992bcb2bd16f7589e 2024-11-10T15:18:14,612 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/04a48bb966594780afbae3e058c933a3 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/04a48bb966594780afbae3e058c933a3 2024-11-10T15:18:14,613 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/6975eb0c7ba64c638696a65811d5ac61 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/6975eb0c7ba64c638696a65811d5ac61 2024-11-10T15:18:14,614 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/cf406ba3b20749429dff4285d96e8504 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/cf406ba3b20749429dff4285d96e8504 2024-11-10T15:18:14,614 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/28d67a32db844c7a9220c1d82468fb35 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/28d67a32db844c7a9220c1d82468fb35 2024-11-10T15:18:14,615 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/99374be1314c4deba6959214d0bc6554 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/99374be1314c4deba6959214d0bc6554 2024-11-10T15:18:14,616 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/1d46dae6a32a46c3b5ed3d292e9038bf to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/1d46dae6a32a46c3b5ed3d292e9038bf 2024-11-10T15:18:14,617 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/4e4eadd0971d4517a2b750a9742b21f8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/4e4eadd0971d4517a2b750a9742b21f8 2024-11-10T15:18:14,617 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/46260ac2490d43029ef4248a3ab6708d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/46260ac2490d43029ef4248a3ab6708d 2024-11-10T15:18:14,618 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/ec1b4b410b95413e87a296e5ce4d3d7a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/ec1b4b410b95413e87a296e5ce4d3d7a 2024-11-10T15:18:14,619 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/854f9c49e418416e95aec5ad03e48d0b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/854f9c49e418416e95aec5ad03e48d0b 2024-11-10T15:18:14,620 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/9257842fa5a64ccd977be9194c719b46 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/9257842fa5a64ccd977be9194c719b46 2024-11-10T15:18:14,621 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/b988868064c44ce28e8c149fdd6b89d6 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/b988868064c44ce28e8c149fdd6b89d6 2024-11-10T15:18:14,622 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/ef6cd149fdd54c75a773529791fedbee to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/ef6cd149fdd54c75a773529791fedbee 2024-11-10T15:18:14,622 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/b839c9b13a904f329be2dce822830938 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/b839c9b13a904f329be2dce822830938 2024-11-10T15:18:14,623 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/1f04e2fed8cb4b0da16f0bd7731b7743 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/1f04e2fed8cb4b0da16f0bd7731b7743 2024-11-10T15:18:14,624 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/75d977f388f5402ca74dea2b258b06dc to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/75d977f388f5402ca74dea2b258b06dc 2024-11-10T15:18:14,625 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/8a6163b924294778a8ec6aff2446c13b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/8a6163b924294778a8ec6aff2446c13b 2024-11-10T15:18:14,626 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/f4c99602a9f4432f9288cc12f88f8c6a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/f4c99602a9f4432f9288cc12f88f8c6a 2024-11-10T15:18:14,627 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/be63bdc3089f424085757c2c42fb926d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/be63bdc3089f424085757c2c42fb926d 2024-11-10T15:18:14,627 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/d4e80bb17fce4760b42869d41a7cff0d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/d4e80bb17fce4760b42869d41a7cff0d 2024-11-10T15:18:14,629 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/5e8b4bc2904c4d92a63670fc309d0568 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/5e8b4bc2904c4d92a63670fc309d0568 2024-11-10T15:18:14,630 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/aa2584595ee94e3cb0a6f9b467badc1d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/aa2584595ee94e3cb0a6f9b467badc1d 2024-11-10T15:18:14,793 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/ff76343e3f614f808925900859900801 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/ff76343e3f614f808925900859900801 2024-11-10T15:18:14,799 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b423d13644c67f79070bddaad3c47156/A of b423d13644c67f79070bddaad3c47156 into ff76343e3f614f808925900859900801(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:14,799 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:14,799 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., storeName=b423d13644c67f79070bddaad3c47156/A, priority=13, startTime=1731251893949; duration=0sec 2024-11-10T15:18:14,799 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:14,799 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b423d13644c67f79070bddaad3c47156:A 2024-11-10T15:18:14,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:14,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T15:18:14,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:18:14,853 INFO [Thread-1616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1616. 2024-11-10T15:18:14,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:14,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:18:14,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:14,853 DEBUG [Thread-1616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:14,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:18:14,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:14,853 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:14,853 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:14,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:14,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d0dd957911334e5b9bc4bdce34bb66af_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251894851/Put/seqid=0 2024-11-10T15:18:14,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742279_1455 (size=12454) 2024-11-10T15:18:14,873 INFO [Thread-1614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1614. 2024-11-10T15:18:14,873 INFO [Thread-1612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1612. 2024-11-10T15:18:14,873 DEBUG [Thread-1614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:14,873 DEBUG [Thread-1612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:14,873 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:14,873 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:14,873 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:14,873 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:14,873 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:14,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:14,883 INFO [Thread-1608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1608. 2024-11-10T15:18:14,883 DEBUG [Thread-1608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:14,883 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:14,883 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:14,883 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:15,263 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:15,267 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d0dd957911334e5b9bc4bdce34bb66af_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d0dd957911334e5b9bc4bdce34bb66af_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:15,267 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/b72c09de42af45349aa4f1844db30e32, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:15,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/b72c09de42af45349aa4f1844db30e32 is 175, key is test_row_0/A:col10/1731251894851/Put/seqid=0 2024-11-10T15:18:15,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742280_1456 (size=31255) 2024-11-10T15:18:15,473 INFO [Thread-1610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1610. 2024-11-10T15:18:15,473 DEBUG [Thread-1610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:15,473 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:15,473 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:15,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:15,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T15:18:15,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-10T15:18:15,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-10T15:18:15,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-10T15:18:15,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-10T15:18:15,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-10T15:18:15,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T15:18:15,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T15:18:15,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 765 2024-11-10T15:18:15,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2295 rows 2024-11-10T15:18:15,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 769 2024-11-10T15:18:15,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2304 rows 2024-11-10T15:18:15,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 751 2024-11-10T15:18:15,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2253 rows 2024-11-10T15:18:15,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 760 2024-11-10T15:18:15,474 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2278 rows 2024-11-10T15:18:15,474 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 761 2024-11-10T15:18:15,474 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2281 rows 2024-11-10T15:18:15,474 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T15:18:15,474 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T15:18:15,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:15,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:15,474 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:15,475 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T15:18:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T15:18:15,480 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251895479"}]},"ts":"1731251895479"} 2024-11-10T15:18:15,482 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T15:18:15,482 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T15:18:15,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T15:18:15,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b423d13644c67f79070bddaad3c47156, UNASSIGN}] 2024-11-10T15:18:15,486 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b423d13644c67f79070bddaad3c47156, UNASSIGN 2024-11-10T15:18:15,486 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=b423d13644c67f79070bddaad3c47156, regionState=CLOSING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:18:15,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b423d13644c67f79070bddaad3c47156, UNASSIGN because future has completed 2024-11-10T15:18:15,488 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T15:18:15,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:18:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T15:18:15,641 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:15,641 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T15:18:15,641 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing b423d13644c67f79070bddaad3c47156, disabling compactions & flushes 2024-11-10T15:18:15,641 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:15,672 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/b72c09de42af45349aa4f1844db30e32 2024-11-10T15:18:15,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/4146760afccf447288dce7140f440f5a is 50, key is test_row_0/B:col10/1731251894851/Put/seqid=0 2024-11-10T15:18:15,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742281_1457 (size=12301) 2024-11-10T15:18:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T15:18:16,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/4146760afccf447288dce7140f440f5a 2024-11-10T15:18:16,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/9a68ac1d9ad242c98e75544f9f3fbc69 is 50, key is test_row_0/C:col10/1731251894851/Put/seqid=0 2024-11-10T15:18:16,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742282_1458 (size=12301) 2024-11-10T15:18:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T15:18:16,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/9a68ac1d9ad242c98e75544f9f3fbc69 2024-11-10T15:18:16,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/b72c09de42af45349aa4f1844db30e32 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/b72c09de42af45349aa4f1844db30e32 2024-11-10T15:18:16,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/b72c09de42af45349aa4f1844db30e32, entries=150, sequenceid=330, filesize=30.5 K 2024-11-10T15:18:16,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/4146760afccf447288dce7140f440f5a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/4146760afccf447288dce7140f440f5a 2024-11-10T15:18:16,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/4146760afccf447288dce7140f440f5a, entries=150, sequenceid=330, filesize=12.0 K 2024-11-10T15:18:16,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/9a68ac1d9ad242c98e75544f9f3fbc69 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/9a68ac1d9ad242c98e75544f9f3fbc69 2024-11-10T15:18:16,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/9a68ac1d9ad242c98e75544f9f3fbc69, entries=150, sequenceid=330, filesize=12.0 K 2024-11-10T15:18:16,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=26.84 KB/27480 for b423d13644c67f79070bddaad3c47156 in 1658ms, sequenceid=330, compaction requested=false 2024-11-10T15:18:16,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b423d13644c67f79070bddaad3c47156: 2024-11-10T15:18:16,510 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:16,510 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:16,510 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. after waiting 0 ms 2024-11-10T15:18:16,510 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:16,510 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing b423d13644c67f79070bddaad3c47156 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-10T15:18:16,511 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=A 2024-11-10T15:18:16,511 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:16,511 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=B 2024-11-10T15:18:16,511 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:16,511 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b423d13644c67f79070bddaad3c47156, store=C 2024-11-10T15:18:16,511 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:16,514 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100e715c8cd749442a92abee5b9addadcd_b423d13644c67f79070bddaad3c47156 is 50, key is test_row_0/A:col10/1731251895471/Put/seqid=0 2024-11-10T15:18:16,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742283_1459 (size=9914) 2024-11-10T15:18:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T15:18:16,918 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:16,926 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100e715c8cd749442a92abee5b9addadcd_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100e715c8cd749442a92abee5b9addadcd_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:16,929 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/c2b359bebb5a49139e646cf82d2725d4, store: [table=TestAcidGuarantees family=A region=b423d13644c67f79070bddaad3c47156] 2024-11-10T15:18:16,930 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/c2b359bebb5a49139e646cf82d2725d4 is 175, key is test_row_0/A:col10/1731251895471/Put/seqid=0 2024-11-10T15:18:16,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742284_1460 (size=22561) 2024-11-10T15:18:17,336 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/c2b359bebb5a49139e646cf82d2725d4 2024-11-10T15:18:17,342 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/83397424597546eabe647d0adc6c5fb0 is 50, key is test_row_0/B:col10/1731251895471/Put/seqid=0 2024-11-10T15:18:17,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742285_1461 (size=9857) 2024-11-10T15:18:17,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T15:18:17,747 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/83397424597546eabe647d0adc6c5fb0 2024-11-10T15:18:17,753 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/2414f78f19214c818555bdd09cfd57c1 is 50, key is test_row_0/C:col10/1731251895471/Put/seqid=0 2024-11-10T15:18:17,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742286_1462 (size=9857) 2024-11-10T15:18:18,156 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/2414f78f19214c818555bdd09cfd57c1 2024-11-10T15:18:18,162 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/A/c2b359bebb5a49139e646cf82d2725d4 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/c2b359bebb5a49139e646cf82d2725d4 2024-11-10T15:18:18,169 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/c2b359bebb5a49139e646cf82d2725d4, entries=100, sequenceid=337, filesize=22.0 K 2024-11-10T15:18:18,179 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/B/83397424597546eabe647d0adc6c5fb0 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/83397424597546eabe647d0adc6c5fb0 2024-11-10T15:18:18,182 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/83397424597546eabe647d0adc6c5fb0, entries=100, sequenceid=337, filesize=9.6 K 2024-11-10T15:18:18,183 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/.tmp/C/2414f78f19214c818555bdd09cfd57c1 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/2414f78f19214c818555bdd09cfd57c1 2024-11-10T15:18:18,189 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/2414f78f19214c818555bdd09cfd57c1, entries=100, sequenceid=337, filesize=9.6 K 2024-11-10T15:18:18,190 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for b423d13644c67f79070bddaad3c47156 in 1679ms, sequenceid=337, compaction requested=true 2024-11-10T15:18:18,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/41a95488893d48ce8ebbc4a47094887b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/8d926be9b32e4d3e8ea7b4e7010ab5ff, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/6d891f6e75174aa5a5e81ae4f2677c10] to archive 2024-11-10T15:18:18,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:18:18,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/41a95488893d48ce8ebbc4a47094887b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/41a95488893d48ce8ebbc4a47094887b 2024-11-10T15:18:18,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/8d926be9b32e4d3e8ea7b4e7010ab5ff to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/8d926be9b32e4d3e8ea7b4e7010ab5ff 2024-11-10T15:18:18,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/6d891f6e75174aa5a5e81ae4f2677c10 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/6d891f6e75174aa5a5e81ae4f2677c10 2024-11-10T15:18:18,214 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/recovered.edits/340.seqid, newMaxSeqId=340, maxSeqId=4 2024-11-10T15:18:18,214 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. 2024-11-10T15:18:18,214 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for b423d13644c67f79070bddaad3c47156: Waiting for close lock at 1731251895641Running coprocessor pre-close hooks at 1731251895641Disabling compacts and flushes for region at 1731251895641Disabling writes for close at 1731251896510 (+869 ms)Obtaining lock to block concurrent updates at 1731251896510Preparing flush snapshotting stores in b423d13644c67f79070bddaad3c47156 at 1731251896510Finished memstore snapshotting TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731251896511 (+1 ms)Flushing stores of TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156. at 1731251896511Flushing b423d13644c67f79070bddaad3c47156/A: creating writer at 1731251896511Flushing mob file b423d13644c67f79070bddaad3c47156/A: appending metadata at 1731251896514 (+3 ms)Flushing mob file b423d13644c67f79070bddaad3c47156/A: closing flushed file at 1731251896514Flushing b423d13644c67f79070bddaad3c47156/A: appending metadata at 1731251896929 (+415 ms)Flushing b423d13644c67f79070bddaad3c47156/A: closing flushed file at 1731251896929Flushing b423d13644c67f79070bddaad3c47156/B: creating writer at 1731251897339 (+410 ms)Flushing b423d13644c67f79070bddaad3c47156/B: appending metadata at 1731251897341 (+2 ms)Flushing b423d13644c67f79070bddaad3c47156/B: closing flushed file at 1731251897341Flushing b423d13644c67f79070bddaad3c47156/C: creating writer at 1731251897750 (+409 ms)Flushing b423d13644c67f79070bddaad3c47156/C: appending metadata at 1731251897752 (+2 ms)Flushing b423d13644c67f79070bddaad3c47156/C: closing flushed file at 1731251897752Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2837afb5: reopening flushed file at 1731251898161 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@433eb295: reopening flushed file at 1731251898169 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3acf8bcf: reopening flushed file at 1731251898183 (+14 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for b423d13644c67f79070bddaad3c47156 in 1679ms, sequenceid=337, compaction requested=true at 1731251898190 (+7 ms)Writing region close event to WAL at 1731251898211 (+21 ms)Running coprocessor post-close hooks at 1731251898214 (+3 ms)Closed at 1731251898214 2024-11-10T15:18:18,217 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:18,217 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=b423d13644c67f79070bddaad3c47156, regionState=CLOSED 2024-11-10T15:18:18,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:18:18,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-10T15:18:18,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure b423d13644c67f79070bddaad3c47156, server=c0771061be61,46087,1731251771284 in 2.7320 sec 2024-11-10T15:18:18,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-10T15:18:18,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b423d13644c67f79070bddaad3c47156, UNASSIGN in 2.7370 sec 2024-11-10T15:18:18,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-10T15:18:18,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.7430 sec 2024-11-10T15:18:18,230 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251898230"}]},"ts":"1731251898230"} 2024-11-10T15:18:18,232 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T15:18:18,233 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T15:18:18,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.7590 sec 2024-11-10T15:18:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T15:18:19,632 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:19,632 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T15:18:19,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:19,634 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:19,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T15:18:19,635 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:19,637 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,639 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/recovered.edits] 2024-11-10T15:18:19,641 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/b72c09de42af45349aa4f1844db30e32 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/b72c09de42af45349aa4f1844db30e32 2024-11-10T15:18:19,642 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/c2b359bebb5a49139e646cf82d2725d4 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/c2b359bebb5a49139e646cf82d2725d4 2024-11-10T15:18:19,642 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/ff76343e3f614f808925900859900801 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/A/ff76343e3f614f808925900859900801 2024-11-10T15:18:19,646 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/4146760afccf447288dce7140f440f5a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/4146760afccf447288dce7140f440f5a 2024-11-10T15:18:19,647 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/48b09b110e0642afb39c028b6edfbf26 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/48b09b110e0642afb39c028b6edfbf26 2024-11-10T15:18:19,647 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/83397424597546eabe647d0adc6c5fb0 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/B/83397424597546eabe647d0adc6c5fb0 2024-11-10T15:18:19,649 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/2414f78f19214c818555bdd09cfd57c1 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/2414f78f19214c818555bdd09cfd57c1 2024-11-10T15:18:19,650 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/9a68ac1d9ad242c98e75544f9f3fbc69 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/9a68ac1d9ad242c98e75544f9f3fbc69 2024-11-10T15:18:19,650 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/ee9a8297d9b14ee595e92779b66732b7 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/C/ee9a8297d9b14ee595e92779b66732b7 2024-11-10T15:18:19,652 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/recovered.edits/340.seqid to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156/recovered.edits/340.seqid 2024-11-10T15:18:19,653 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,653 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T15:18:19,653 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T15:18:19,653 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T15:18:19,656 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100e715c8cd749442a92abee5b9addadcd_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100e715c8cd749442a92abee5b9addadcd_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,656 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110149ca832560340f8b49dd850c0a06b75_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110149ca832560340f8b49dd850c0a06b75_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,657 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111014d2868335754603af8eda740b2cb97f_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111014d2868335754603af8eda740b2cb97f_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,658 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111027a9cdd74e7e4fefa880f829c5e5e00e_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111027a9cdd74e7e4fefa880f829c5e5e00e_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,659 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111037db3eb39b4645aab064c52d1136c13b_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111037db3eb39b4645aab064c52d1136c13b_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,659 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103db4f0bb9e6041db833af8bf8a8f67f5_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103db4f0bb9e6041db833af8bf8a8f67f5_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,660 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104fd654970c4047b29ca68ebfdff405a1_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104fd654970c4047b29ca68ebfdff405a1_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,661 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111057e14cc2dbdb4a078f830ed521dcec47_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111057e14cc2dbdb4a078f830ed521dcec47_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,662 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110742c52f005534ae687806e3dfa4ab900_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110742c52f005534ae687806e3dfa4ab900_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,663 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110774c81abd7314acbaeaeca4f7841df58_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110774c81abd7314acbaeaeca4f7841df58_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,663 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107bd0996a3c4c47778eda1d44779a6dd6_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107bd0996a3c4c47778eda1d44779a6dd6_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,664 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109258771336ee4f689d323bef40a416ca_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109258771336ee4f689d323bef40a416ca_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,665 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a25b5679834545ba9e4527ee81604921_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a25b5679834545ba9e4527ee81604921_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,666 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a33488c3593a4aa8a35a4fe4aad57f86_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a33488c3593a4aa8a35a4fe4aad57f86_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,667 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ae1a4351a7474967b3a0908a12943c62_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ae1a4351a7474967b3a0908a12943c62_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,667 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b1624a3258c140b8ac139d9efeaaf83d_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b1624a3258c140b8ac139d9efeaaf83d_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,668 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d0dd957911334e5b9bc4bdce34bb66af_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d0dd957911334e5b9bc4bdce34bb66af_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,669 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d9a56d61febb4040989a6f63285aa329_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d9a56d61febb4040989a6f63285aa329_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,670 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ebc326bcbef646b896ee8774d29308c5_b423d13644c67f79070bddaad3c47156 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ebc326bcbef646b896ee8774d29308c5_b423d13644c67f79070bddaad3c47156 2024-11-10T15:18:19,670 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T15:18:19,672 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:19,675 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T15:18:19,676 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T15:18:19,678 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:19,678 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T15:18:19,678 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731251899678"}]},"ts":"9223372036854775807"} 2024-11-10T15:18:19,680 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T15:18:19,680 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b423d13644c67f79070bddaad3c47156, NAME => 'TestAcidGuarantees,,1731251869558.b423d13644c67f79070bddaad3c47156.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T15:18:19,680 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T15:18:19,680 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731251899680"}]},"ts":"9223372036854775807"} 2024-11-10T15:18:19,682 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T15:18:19,683 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:19,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-10T15:18:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T15:18:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T15:18:19,744 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:19,751 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 246) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/c0771061be61:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2087031749_22 at /127.0.0.1:45516 [Waiting for operation #1030] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1405295892_22 at /127.0.0.1:45762 [Waiting for operation #762] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1405295892_22 at /127.0.0.1:45474 [Waiting for operation #1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2087031749_22 at /127.0.0.1:33894 [Waiting for operation #1078] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=946 (was 678) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3640 (was 3729) 2024-11-10T15:18:19,761 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=946, ProcessCount=11, AvailableMemoryMB=3640 2024-11-10T15:18:19,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T15:18:19,762 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T15:18:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:19,764 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T15:18:19,765 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:19,765 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-10T15:18:19,765 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T15:18:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-10T15:18:19,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742287_1463 (size=1061) 2024-11-10T15:18:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-10T15:18:20,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-10T15:18:20,174 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85 2024-11-10T15:18:20,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742288_1464 (size=53) 2024-11-10T15:18:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-10T15:18:20,582 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:18:20,582 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ba35a6e7578835b556d2567ed9c44ec8, disabling compactions & flushes 2024-11-10T15:18:20,582 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:20,582 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:20,582 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. after waiting 0 ms 2024-11-10T15:18:20,582 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:20,582 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:20,582 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ba35a6e7578835b556d2567ed9c44ec8: Waiting for close lock at 1731251900582Disabling compacts and flushes for region at 1731251900582Disabling writes for close at 1731251900582Writing region close event to WAL at 1731251900582Closed at 1731251900582 2024-11-10T15:18:20,584 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T15:18:20,584 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731251900584"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731251900584"}]},"ts":"1731251900584"} 2024-11-10T15:18:20,586 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T15:18:20,587 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T15:18:20,587 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251900587"}]},"ts":"1731251900587"} 2024-11-10T15:18:20,589 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T15:18:20,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba35a6e7578835b556d2567ed9c44ec8, ASSIGN}] 2024-11-10T15:18:20,590 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba35a6e7578835b556d2567ed9c44ec8, ASSIGN 2024-11-10T15:18:20,591 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba35a6e7578835b556d2567ed9c44ec8, ASSIGN; state=OFFLINE, location=c0771061be61,46087,1731251771284; forceNewPlan=false, retain=false 2024-11-10T15:18:20,742 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=ba35a6e7578835b556d2567ed9c44ec8, regionState=OPENING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:18:20,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba35a6e7578835b556d2567ed9c44ec8, ASSIGN because future has completed 2024-11-10T15:18:20,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:18:20,884 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T15:18:20,897 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:20,898 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} 2024-11-10T15:18:20,899 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:20,899 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:18:20,899 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:20,899 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:20,900 INFO [StoreOpener-ba35a6e7578835b556d2567ed9c44ec8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:20,901 INFO [StoreOpener-ba35a6e7578835b556d2567ed9c44ec8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:18:20,901 INFO [StoreOpener-ba35a6e7578835b556d2567ed9c44ec8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba35a6e7578835b556d2567ed9c44ec8 columnFamilyName A 2024-11-10T15:18:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-10T15:18:20,901 DEBUG [StoreOpener-ba35a6e7578835b556d2567ed9c44ec8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:20,902 INFO [StoreOpener-ba35a6e7578835b556d2567ed9c44ec8-1 {}] regionserver.HStore(327): Store=ba35a6e7578835b556d2567ed9c44ec8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:18:20,902 INFO [StoreOpener-ba35a6e7578835b556d2567ed9c44ec8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:20,903 INFO [StoreOpener-ba35a6e7578835b556d2567ed9c44ec8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:18:20,903 INFO [StoreOpener-ba35a6e7578835b556d2567ed9c44ec8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba35a6e7578835b556d2567ed9c44ec8 columnFamilyName B 2024-11-10T15:18:20,903 DEBUG [StoreOpener-ba35a6e7578835b556d2567ed9c44ec8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:20,903 INFO [StoreOpener-ba35a6e7578835b556d2567ed9c44ec8-1 {}] regionserver.HStore(327): Store=ba35a6e7578835b556d2567ed9c44ec8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:18:20,903 INFO [StoreOpener-ba35a6e7578835b556d2567ed9c44ec8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:20,904 INFO [StoreOpener-ba35a6e7578835b556d2567ed9c44ec8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:18:20,904 INFO [StoreOpener-ba35a6e7578835b556d2567ed9c44ec8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba35a6e7578835b556d2567ed9c44ec8 columnFamilyName C 2024-11-10T15:18:20,904 DEBUG [StoreOpener-ba35a6e7578835b556d2567ed9c44ec8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:20,905 INFO [StoreOpener-ba35a6e7578835b556d2567ed9c44ec8-1 {}] regionserver.HStore(327): Store=ba35a6e7578835b556d2567ed9c44ec8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:18:20,905 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:20,905 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:20,906 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:20,906 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:20,907 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:20,907 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:20,907 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T15:18:20,908 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:20,910 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T15:18:20,910 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened ba35a6e7578835b556d2567ed9c44ec8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65412774, jitterRate=-0.02527371048927307}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T15:18:20,910 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:20,911 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for ba35a6e7578835b556d2567ed9c44ec8: Running coprocessor pre-open hook at 1731251900899Writing region info on filesystem at 1731251900899Initializing all the Stores at 1731251900899Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251900899Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251900900 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251900900Cleaning up temporary data from old regions at 1731251900907 (+7 ms)Running coprocessor post-open hooks at 1731251900910 (+3 ms)Region opened successfully at 1731251900911 (+1 ms) 2024-11-10T15:18:20,912 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., pid=120, masterSystemTime=1731251900895 2024-11-10T15:18:20,913 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:20,913 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:20,914 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=ba35a6e7578835b556d2567ed9c44ec8, regionState=OPEN, openSeqNum=2, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:18:20,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:18:20,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-10T15:18:20,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 in 173 msec 2024-11-10T15:18:20,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-10T15:18:20,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba35a6e7578835b556d2567ed9c44ec8, ASSIGN in 328 msec 2024-11-10T15:18:20,919 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T15:18:20,919 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251900919"}]},"ts":"1731251900919"} 2024-11-10T15:18:20,920 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T15:18:20,921 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T15:18:20,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1590 sec 2024-11-10T15:18:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-10T15:18:21,912 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:21,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:21,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:21,914 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:21,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:21,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a0d162b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:21,916 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c3c99d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,917 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:21,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:21,920 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T15:18:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is c0771061be61,41501,1731251770419 2024-11-10T15:18:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7d5dd911 2024-11-10T15:18:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T15:18:21,922 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T15:18:21,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,924 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:21,924 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:21,924 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dbcf130, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:21,926 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76b3af30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,927 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:21,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:21,928 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d293d0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:21,929 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:21,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b90544d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,930 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:21,931 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:21,931 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:21,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:21,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:21,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aa7df2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:21,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:21,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:21,933 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f1b31e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:21,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:21,935 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18f0388b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:21,936 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5aaa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:21,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:21,941 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d81792c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:21,943 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62b3ad43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:21,944 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:21,945 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce53aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:21,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:21,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:21,946 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:21,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6597097c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,948 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:21,948 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:21,948 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4293c27b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:21,950 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c077fc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:21,951 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:21,952 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20822bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:21,953 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@455c2523, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,955 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:21,955 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:21,955 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:21,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:21,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:21,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d236c21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:21,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:21,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:21,957 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:21,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@346255ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663bb5d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:21,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:21,959 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45bae53a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:21,962 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@462d6284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:21,964 DEBUG [Thread-2077 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:21,964 DEBUG [Thread-2092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:21,964 DEBUG [Thread-2088 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:21,965 DEBUG [Thread-2092 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:21,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:21,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:21,966 DEBUG [Thread-2092 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:21,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:21,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:18:21,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:18:21,968 DEBUG [Thread-2079 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:21,971 DEBUG [Thread-2092 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:18:21,972 DEBUG [Thread-2081 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:21,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:21,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-10T15:18:21,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:21,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:21,980 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:21,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:18:21,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T15:18:21,982 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:21,982 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:21,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:21,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:18:21,986 DEBUG [Thread-2094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:21,987 DEBUG [Thread-2094 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:21,987 DEBUG [Thread-2094 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:21,988 DEBUG [Thread-2083 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:21,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:21,989 DEBUG [Thread-2090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:18:21,994 DEBUG [Thread-2085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:21,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:21,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:21,996 DEBUG [Thread-2096 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:21,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:22,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:18:22,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T15:18:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:22,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:22,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:22,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:22,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:22,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:22,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:22,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:22,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:22,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:22,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:22,005 DEBUG [Thread-2096 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:22,005 DEBUG [Thread-2096 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:22,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:22,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:22,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:22,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:18:22,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:18:22,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2] 2024-11-10T15:18:22,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/c9a01ecb6a264642b94ee7c91065132a is 50, key is test_row_0/A:col10/1731251901993/Put/seqid=0 2024-11-10T15:18:22,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251962051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251962053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742289_1465 (size=14341) 2024-11-10T15:18:22,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/c9a01ecb6a264642b94ee7c91065132a 2024-11-10T15:18:22,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251962069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251962070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251962074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T15:18:22,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/9bdb1f75328844df9fd4cf3a125c35c3 is 50, key is test_row_0/B:col10/1731251901993/Put/seqid=0 2024-11-10T15:18:22,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742290_1466 (size=12001) 2024-11-10T15:18:22,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/9bdb1f75328844df9fd4cf3a125c35c3 2024-11-10T15:18:22,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T15:18:22,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:22,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:22,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:22,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:22,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:22,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/a587d086b09c4919a3eb718be7dae653 is 50, key is test_row_0/C:col10/1731251901993/Put/seqid=0 2024-11-10T15:18:22,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251962183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251962183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251962192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742291_1467 (size=12001) 2024-11-10T15:18:22,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/a587d086b09c4919a3eb718be7dae653 2024-11-10T15:18:22,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251962202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251962202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/c9a01ecb6a264642b94ee7c91065132a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/c9a01ecb6a264642b94ee7c91065132a 2024-11-10T15:18:22,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/c9a01ecb6a264642b94ee7c91065132a, entries=200, sequenceid=13, filesize=14.0 K 2024-11-10T15:18:22,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/9bdb1f75328844df9fd4cf3a125c35c3 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/9bdb1f75328844df9fd4cf3a125c35c3 2024-11-10T15:18:22,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/9bdb1f75328844df9fd4cf3a125c35c3, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T15:18:22,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/a587d086b09c4919a3eb718be7dae653 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/a587d086b09c4919a3eb718be7dae653 2024-11-10T15:18:22,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/a587d086b09c4919a3eb718be7dae653, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T15:18:22,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ba35a6e7578835b556d2567ed9c44ec8 in 229ms, sequenceid=13, compaction requested=false 2024-11-10T15:18:22,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:22,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T15:18:22,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:22,291 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T15:18:22,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:22,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:22,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:22,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:22,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:22,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:22,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/ecf676fb6db94cbdb758bdd3f3a3b03c is 50, key is test_row_0/A:col10/1731251902071/Put/seqid=0 2024-11-10T15:18:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T15:18:22,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742292_1468 (size=12001) 2024-11-10T15:18:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:22,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:22,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251962419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251962420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251962421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251962422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251962423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251962531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251962541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251962541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251962541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251962542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T15:18:22,724 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/ecf676fb6db94cbdb758bdd3f3a3b03c 2024-11-10T15:18:22,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/683bd55a977c40f69dbc8470884e59bf is 50, key is test_row_0/B:col10/1731251902071/Put/seqid=0 2024-11-10T15:18:22,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251962741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742293_1469 (size=12001) 2024-11-10T15:18:22,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251962751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251962751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251962751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251962751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251963061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251963072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251963072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251963072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251963072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T15:18:23,156 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/683bd55a977c40f69dbc8470884e59bf 2024-11-10T15:18:23,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/b01050292be040849bc1bf413cbdbeae is 50, key is test_row_0/C:col10/1731251902071/Put/seqid=0 2024-11-10T15:18:23,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742294_1470 (size=12001) 2024-11-10T15:18:23,254 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/b01050292be040849bc1bf413cbdbeae 2024-11-10T15:18:23,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/ecf676fb6db94cbdb758bdd3f3a3b03c as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/ecf676fb6db94cbdb758bdd3f3a3b03c 2024-11-10T15:18:23,267 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/ecf676fb6db94cbdb758bdd3f3a3b03c, entries=150, sequenceid=33, filesize=11.7 K 2024-11-10T15:18:23,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/683bd55a977c40f69dbc8470884e59bf as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/683bd55a977c40f69dbc8470884e59bf 2024-11-10T15:18:23,274 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/683bd55a977c40f69dbc8470884e59bf, entries=150, sequenceid=33, filesize=11.7 K 2024-11-10T15:18:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/b01050292be040849bc1bf413cbdbeae as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/b01050292be040849bc1bf413cbdbeae 2024-11-10T15:18:23,287 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/b01050292be040849bc1bf413cbdbeae, entries=150, sequenceid=33, filesize=11.7 K 2024-11-10T15:18:23,287 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ba35a6e7578835b556d2567ed9c44ec8 in 996ms, sequenceid=33, compaction requested=false 2024-11-10T15:18:23,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:23,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:23,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-10T15:18:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-10T15:18:23,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-10T15:18:23,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3170 sec 2024-11-10T15:18:23,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.3270 sec 2024-11-10T15:18:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:23,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T15:18:23,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:23,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:23,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:23,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:23,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:23,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:23,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/0da2442176a54db3802103b2827cb0eb is 50, key is test_row_0/A:col10/1731251902421/Put/seqid=0 2024-11-10T15:18:23,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742295_1471 (size=14341) 2024-11-10T15:18:23,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251963614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251963615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251963616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251963625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251963630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,743 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T15:18:23,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251963742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251963743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251963743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251963743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251963759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251963961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251963973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251963980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251963980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251963982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:23,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/0da2442176a54db3802103b2827cb0eb 2024-11-10T15:18:24,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/aa3aef0518704095abbf5564f5b3f0ef is 50, key is test_row_0/B:col10/1731251902421/Put/seqid=0 2024-11-10T15:18:24,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742296_1472 (size=12001) 2024-11-10T15:18:24,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/aa3aef0518704095abbf5564f5b3f0ef 2024-11-10T15:18:24,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/54fff8a76e614663ac91eec637cf2183 is 50, key is test_row_0/C:col10/1731251902421/Put/seqid=0 2024-11-10T15:18:24,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742297_1473 (size=12001) 2024-11-10T15:18:24,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/54fff8a76e614663ac91eec637cf2183 2024-11-10T15:18:24,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/0da2442176a54db3802103b2827cb0eb as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/0da2442176a54db3802103b2827cb0eb 2024-11-10T15:18:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T15:18:24,132 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:24,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-10T15:18:24,135 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:24,136 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T15:18:24,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:24,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/0da2442176a54db3802103b2827cb0eb, entries=200, sequenceid=46, filesize=14.0 K 2024-11-10T15:18:24,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/aa3aef0518704095abbf5564f5b3f0ef as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/aa3aef0518704095abbf5564f5b3f0ef 2024-11-10T15:18:24,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/aa3aef0518704095abbf5564f5b3f0ef, entries=150, sequenceid=46, filesize=11.7 K 2024-11-10T15:18:24,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/54fff8a76e614663ac91eec637cf2183 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/54fff8a76e614663ac91eec637cf2183 2024-11-10T15:18:24,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/54fff8a76e614663ac91eec637cf2183, entries=150, sequenceid=46, filesize=11.7 K 2024-11-10T15:18:24,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ba35a6e7578835b556d2567ed9c44ec8 in 568ms, sequenceid=46, compaction requested=true 2024-11-10T15:18:24,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:24,154 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:24,159 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:24,159 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/A is initiating minor compaction (all files) 2024-11-10T15:18:24,159 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/A in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:24,159 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/c9a01ecb6a264642b94ee7c91065132a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/ecf676fb6db94cbdb758bdd3f3a3b03c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/0da2442176a54db3802103b2827cb0eb] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=39.7 K 2024-11-10T15:18:24,160 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9a01ecb6a264642b94ee7c91065132a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731251901991 2024-11-10T15:18:24,160 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecf676fb6db94cbdb758bdd3f3a3b03c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731251902049 2024-11-10T15:18:24,161 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0da2442176a54db3802103b2827cb0eb, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731251902420 2024-11-10T15:18:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:24,167 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:24,169 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:24,169 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/B is initiating minor compaction (all files) 2024-11-10T15:18:24,169 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/B in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:24,169 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/9bdb1f75328844df9fd4cf3a125c35c3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/683bd55a977c40f69dbc8470884e59bf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/aa3aef0518704095abbf5564f5b3f0ef] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=35.2 K 2024-11-10T15:18:24,177 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bdb1f75328844df9fd4cf3a125c35c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731251901991 2024-11-10T15:18:24,177 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 683bd55a977c40f69dbc8470884e59bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731251902049 2024-11-10T15:18:24,177 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting aa3aef0518704095abbf5564f5b3f0ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731251902421 2024-11-10T15:18:24,195 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#B#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:24,195 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/de0a15de73174c7491ffdb0e34acb337 is 50, key is test_row_0/B:col10/1731251902421/Put/seqid=0 2024-11-10T15:18:24,196 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#A#compaction#388 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:24,197 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/da7ed475cff444edacb40a47ca3db1e4 is 50, key is test_row_0/A:col10/1731251902421/Put/seqid=0 2024-11-10T15:18:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T15:18:24,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742298_1474 (size=12104) 2024-11-10T15:18:24,268 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/de0a15de73174c7491ffdb0e34acb337 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/de0a15de73174c7491ffdb0e34acb337 2024-11-10T15:18:24,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742299_1475 (size=12104) 2024-11-10T15:18:24,276 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/B of ba35a6e7578835b556d2567ed9c44ec8 into de0a15de73174c7491ffdb0e34acb337(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:24,277 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:24,277 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/B, priority=13, startTime=1731251904167; duration=0sec 2024-11-10T15:18:24,277 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:24,277 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:B 2024-11-10T15:18:24,277 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:24,279 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:24,279 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/C is initiating minor compaction (all files) 2024-11-10T15:18:24,279 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/C in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:24,279 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/a587d086b09c4919a3eb718be7dae653, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/b01050292be040849bc1bf413cbdbeae, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/54fff8a76e614663ac91eec637cf2183] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=35.2 K 2024-11-10T15:18:24,280 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting a587d086b09c4919a3eb718be7dae653, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731251901991 2024-11-10T15:18:24,287 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting b01050292be040849bc1bf413cbdbeae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731251902049 2024-11-10T15:18:24,290 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 54fff8a76e614663ac91eec637cf2183, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731251902421 2024-11-10T15:18:24,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T15:18:24,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:24,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:24,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:24,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:24,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:24,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:24,292 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/da7ed475cff444edacb40a47ca3db1e4 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/da7ed475cff444edacb40a47ca3db1e4 2024-11-10T15:18:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:24,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T15:18:24,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:24,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:24,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:24,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:24,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:24,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:24,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/419d61242cde4e818533f0d4ee01d0b1 is 50, key is test_row_0/A:col10/1731251903615/Put/seqid=0 2024-11-10T15:18:24,310 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/A of ba35a6e7578835b556d2567ed9c44ec8 into da7ed475cff444edacb40a47ca3db1e4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:24,310 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:24,310 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/A, priority=13, startTime=1731251904154; duration=0sec 2024-11-10T15:18:24,310 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:24,310 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:A 2024-11-10T15:18:24,314 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#C#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:24,315 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/5608ca45d8894f6d8a30f1d367ab7301 is 50, key is test_row_0/C:col10/1731251902421/Put/seqid=0 2024-11-10T15:18:24,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251964320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251964323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251964324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251964327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251964335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742301_1477 (size=12104) 2024-11-10T15:18:24,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742300_1476 (size=14341) 2024-11-10T15:18:24,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/419d61242cde4e818533f0d4ee01d0b1 2024-11-10T15:18:24,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/56e65dd4e8884de39820425c28f9c144 is 50, key is test_row_0/B:col10/1731251903615/Put/seqid=0 2024-11-10T15:18:24,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251964442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251964442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251964442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T15:18:24,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T15:18:24,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:24,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:24,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:24,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:24,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:24,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251964454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251964456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742302_1478 (size=12001) 2024-11-10T15:18:24,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T15:18:24,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:24,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:24,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:24,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:24,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:24,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251964661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251964661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251964671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251964671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251964671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T15:18:24,761 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/5608ca45d8894f6d8a30f1d367ab7301 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/5608ca45d8894f6d8a30f1d367ab7301 2024-11-10T15:18:24,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T15:18:24,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:24,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:24,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:24,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:24,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:24,765 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/C of ba35a6e7578835b556d2567ed9c44ec8 into 5608ca45d8894f6d8a30f1d367ab7301(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:24,765 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:24,765 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/C, priority=13, startTime=1731251904167; duration=0sec 2024-11-10T15:18:24,766 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:24,766 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:C 2024-11-10T15:18:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:24,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/56e65dd4e8884de39820425c28f9c144 2024-11-10T15:18:24,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/125830b1fbed4e2895ac5aea96702a02 is 50, key is test_row_0/C:col10/1731251903615/Put/seqid=0 2024-11-10T15:18:24,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742303_1479 (size=12001) 2024-11-10T15:18:24,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T15:18:24,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:24,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:24,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:24,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:24,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:24,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251964971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251964971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251964991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251964991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251964991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:25,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T15:18:25,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:25,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:25,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:25,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:25,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:25,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T15:18:25,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:25,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:25,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:25,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:25,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:25,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:25,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T15:18:25,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/125830b1fbed4e2895ac5aea96702a02 2024-11-10T15:18:25,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/419d61242cde4e818533f0d4ee01d0b1 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/419d61242cde4e818533f0d4ee01d0b1 2024-11-10T15:18:25,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/419d61242cde4e818533f0d4ee01d0b1, entries=200, sequenceid=67, filesize=14.0 K 2024-11-10T15:18:25,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/56e65dd4e8884de39820425c28f9c144 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/56e65dd4e8884de39820425c28f9c144 2024-11-10T15:18:25,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/56e65dd4e8884de39820425c28f9c144, entries=150, sequenceid=67, filesize=11.7 K 2024-11-10T15:18:25,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/125830b1fbed4e2895ac5aea96702a02 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/125830b1fbed4e2895ac5aea96702a02 2024-11-10T15:18:25,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/125830b1fbed4e2895ac5aea96702a02, entries=150, sequenceid=67, filesize=11.7 K 2024-11-10T15:18:25,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ba35a6e7578835b556d2567ed9c44ec8 in 1018ms, sequenceid=67, compaction requested=false 2024-11-10T15:18:25,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:25,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T15:18:25,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:25,377 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T15:18:25,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:25,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:25,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:25,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:25,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:25,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:25,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/529775cc588d430397c09ae9b6fd36e8 is 50, key is test_row_0/A:col10/1731251904333/Put/seqid=0 2024-11-10T15:18:25,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742304_1480 (size=12001) 2024-11-10T15:18:25,385 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/529775cc588d430397c09ae9b6fd36e8 2024-11-10T15:18:25,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/d81a871d6473490fbd5b8f90209c1080 is 50, key is test_row_0/B:col10/1731251904333/Put/seqid=0 2024-11-10T15:18:25,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742305_1481 (size=12001) 2024-11-10T15:18:25,418 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/d81a871d6473490fbd5b8f90209c1080 2024-11-10T15:18:25,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/0b933cc3cea249b791bb8f1b5b9633d0 is 50, key is test_row_0/C:col10/1731251904333/Put/seqid=0 2024-11-10T15:18:25,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742306_1482 (size=12001) 2024-11-10T15:18:25,441 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/0b933cc3cea249b791bb8f1b5b9633d0 2024-11-10T15:18:25,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/529775cc588d430397c09ae9b6fd36e8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/529775cc588d430397c09ae9b6fd36e8 2024-11-10T15:18:25,461 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/529775cc588d430397c09ae9b6fd36e8, entries=150, sequenceid=81, filesize=11.7 K 2024-11-10T15:18:25,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/d81a871d6473490fbd5b8f90209c1080 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d81a871d6473490fbd5b8f90209c1080 2024-11-10T15:18:25,467 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d81a871d6473490fbd5b8f90209c1080, entries=150, sequenceid=81, filesize=11.7 K 2024-11-10T15:18:25,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/0b933cc3cea249b791bb8f1b5b9633d0 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0b933cc3cea249b791bb8f1b5b9633d0 2024-11-10T15:18:25,471 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0b933cc3cea249b791bb8f1b5b9633d0, entries=150, sequenceid=81, filesize=11.7 K 2024-11-10T15:18:25,472 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for ba35a6e7578835b556d2567ed9c44ec8 in 95ms, sequenceid=81, compaction requested=true 2024-11-10T15:18:25,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:25,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:25,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-10T15:18:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-10T15:18:25,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-10T15:18:25,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3370 sec 2024-11-10T15:18:25,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.3430 sec 2024-11-10T15:18:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:25,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T15:18:25,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:25,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:25,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:25,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:25,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:25,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:25,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/b47c90e9daec4d55b59adcf3c15ca11a is 50, key is test_row_0/A:col10/1731251905495/Put/seqid=0 2024-11-10T15:18:25,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742307_1483 (size=14341) 2024-11-10T15:18:25,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251965566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:25,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:25,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731251905560, max timestamp=1731251905586Num uniques 859; 2024-11-10T15:18:25,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T15:18:25,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731251905560, max timestamp=1731251905586Num uniques 859; 2024-11-10T15:18:25,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731251905560, max timestamp=1731251905586Num uniques 859; 2024-11-10T15:18:25,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T15:18:25,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T15:18:25,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251965632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:25,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251965634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:25,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251965639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251965640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:25,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251965693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:25,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251965752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:25,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251965752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:25,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251965761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:25,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251965761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:25,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:25,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251965911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:25,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:25,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/b47c90e9daec4d55b59adcf3c15ca11a 2024-11-10T15:18:25,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/dc5d33b203b14ca4a5edc7a5b41bd172 is 50, key is test_row_0/B:col10/1731251905495/Put/seqid=0 2024-11-10T15:18:25,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742308_1484 (size=12001) 2024-11-10T15:18:25,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251965971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:25,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251965981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:25,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:25,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251965981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:25,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:25,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251965981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:26,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251966231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T15:18:26,281 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:26,281 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-10T15:18:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T15:18:26,283 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:26,283 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:26,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:26,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251966296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:26,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:26,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251966301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251966301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:26,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251966301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:26,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/dc5d33b203b14ca4a5edc7a5b41bd172 2024-11-10T15:18:26,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/cabe29218f364b59a96a955328796578 is 50, key is test_row_0/C:col10/1731251905495/Put/seqid=0 2024-11-10T15:18:26,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742309_1485 (size=12001) 2024-11-10T15:18:26,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/cabe29218f364b59a96a955328796578 2024-11-10T15:18:26,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/b47c90e9daec4d55b59adcf3c15ca11a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/b47c90e9daec4d55b59adcf3c15ca11a 2024-11-10T15:18:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T15:18:26,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/b47c90e9daec4d55b59adcf3c15ca11a, entries=200, sequenceid=91, filesize=14.0 K 2024-11-10T15:18:26,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/dc5d33b203b14ca4a5edc7a5b41bd172 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/dc5d33b203b14ca4a5edc7a5b41bd172 2024-11-10T15:18:26,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/dc5d33b203b14ca4a5edc7a5b41bd172, entries=150, sequenceid=91, filesize=11.7 K 2024-11-10T15:18:26,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/cabe29218f364b59a96a955328796578 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/cabe29218f364b59a96a955328796578 2024-11-10T15:18:26,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/cabe29218f364b59a96a955328796578, entries=150, sequenceid=91, filesize=11.7 K 2024-11-10T15:18:26,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ba35a6e7578835b556d2567ed9c44ec8 in 881ms, sequenceid=91, compaction requested=true 2024-11-10T15:18:26,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:26,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:26,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:26,410 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:18:26,410 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:18:26,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:26,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:26,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:26,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:26,412 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:18:26,412 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/A is initiating minor compaction (all files) 2024-11-10T15:18:26,412 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/A in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:26,412 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:18:26,412 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/B is initiating minor compaction (all files) 2024-11-10T15:18:26,412 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/da7ed475cff444edacb40a47ca3db1e4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/419d61242cde4e818533f0d4ee01d0b1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/529775cc588d430397c09ae9b6fd36e8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/b47c90e9daec4d55b59adcf3c15ca11a] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=51.5 K 2024-11-10T15:18:26,412 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/B in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:26,412 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/de0a15de73174c7491ffdb0e34acb337, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/56e65dd4e8884de39820425c28f9c144, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d81a871d6473490fbd5b8f90209c1080, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/dc5d33b203b14ca4a5edc7a5b41bd172] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=47.0 K 2024-11-10T15:18:26,415 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting da7ed475cff444edacb40a47ca3db1e4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731251902421 2024-11-10T15:18:26,415 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting de0a15de73174c7491ffdb0e34acb337, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731251902421 2024-11-10T15:18:26,415 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 419d61242cde4e818533f0d4ee01d0b1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731251903611 2024-11-10T15:18:26,415 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 56e65dd4e8884de39820425c28f9c144, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731251903611 2024-11-10T15:18:26,415 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 529775cc588d430397c09ae9b6fd36e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731251904319 2024-11-10T15:18:26,415 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting d81a871d6473490fbd5b8f90209c1080, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731251904319 2024-11-10T15:18:26,416 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting dc5d33b203b14ca4a5edc7a5b41bd172, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731251905495 2024-11-10T15:18:26,416 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b47c90e9daec4d55b59adcf3c15ca11a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731251905495 2024-11-10T15:18:26,431 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#B#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:26,431 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/247197642bc64224b3384f3cd4045f87 is 50, key is test_row_0/B:col10/1731251905495/Put/seqid=0 2024-11-10T15:18:26,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T15:18:26,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:26,436 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-10T15:18:26,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:26,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:26,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:26,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:26,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:26,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:26,444 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#A#compaction#400 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:26,444 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/0644c7032d9548ce98985ea91ef1dd00 is 50, key is test_row_0/A:col10/1731251905495/Put/seqid=0 2024-11-10T15:18:26,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/a54f8d83ece44871be0fbbcd5423d4fa is 50, key is test_row_0/A:col10/1731251905615/Put/seqid=0 2024-11-10T15:18:26,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742311_1487 (size=12241) 2024-11-10T15:18:26,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742312_1488 (size=12001) 2024-11-10T15:18:26,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742310_1486 (size=12241) 2024-11-10T15:18:26,544 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/0644c7032d9548ce98985ea91ef1dd00 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/0644c7032d9548ce98985ea91ef1dd00 2024-11-10T15:18:26,569 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/A of ba35a6e7578835b556d2567ed9c44ec8 into 0644c7032d9548ce98985ea91ef1dd00(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:26,569 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:26,570 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/A, priority=12, startTime=1731251906410; duration=0sec 2024-11-10T15:18:26,570 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:26,570 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:A 2024-11-10T15:18:26,570 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:18:26,580 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:18:26,580 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/C is initiating minor compaction (all files) 2024-11-10T15:18:26,580 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/C in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:26,580 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/5608ca45d8894f6d8a30f1d367ab7301, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/125830b1fbed4e2895ac5aea96702a02, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0b933cc3cea249b791bb8f1b5b9633d0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/cabe29218f364b59a96a955328796578] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=47.0 K 2024-11-10T15:18:26,584 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5608ca45d8894f6d8a30f1d367ab7301, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731251902421 2024-11-10T15:18:26,588 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 125830b1fbed4e2895ac5aea96702a02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731251903611 2024-11-10T15:18:26,589 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b933cc3cea249b791bb8f1b5b9633d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731251904319 2024-11-10T15:18:26,596 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting cabe29218f364b59a96a955328796578, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731251905495 2024-11-10T15:18:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T15:18:26,629 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#C#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:26,630 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/f62a066ea8544ba688196205e3f30c3f is 50, key is test_row_0/C:col10/1731251905495/Put/seqid=0 2024-11-10T15:18:26,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742313_1489 (size=12241) 2024-11-10T15:18:26,679 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/f62a066ea8544ba688196205e3f30c3f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/f62a066ea8544ba688196205e3f30c3f 2024-11-10T15:18:26,724 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/C of ba35a6e7578835b556d2567ed9c44ec8 into f62a066ea8544ba688196205e3f30c3f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:26,724 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:26,724 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/C, priority=12, startTime=1731251906410; duration=0sec 2024-11-10T15:18:26,724 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:26,724 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:C 2024-11-10T15:18:26,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:26,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251966822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:26,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:26,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251966822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:26,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:26,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251966822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:26,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:26,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251966822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:26,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251966836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T15:18:26,927 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/a54f8d83ece44871be0fbbcd5423d4fa 2024-11-10T15:18:26,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/5c6dd6d279c24a359f581e390a0d683a is 50, key is test_row_0/B:col10/1731251905615/Put/seqid=0 2024-11-10T15:18:26,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742314_1490 (size=12001) 2024-11-10T15:18:26,950 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/247197642bc64224b3384f3cd4045f87 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/247197642bc64224b3384f3cd4045f87 2024-11-10T15:18:26,954 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/B of ba35a6e7578835b556d2567ed9c44ec8 into 247197642bc64224b3384f3cd4045f87(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:26,954 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:26,954 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/B, priority=12, startTime=1731251906410; duration=0sec 2024-11-10T15:18:26,954 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:26,954 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:B 2024-11-10T15:18:26,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251966961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:27,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251967181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:27,338 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/5c6dd6d279c24a359f581e390a0d683a 2024-11-10T15:18:27,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/d5893eca73d0448d833409464b241a3a is 50, key is test_row_0/C:col10/1731251905615/Put/seqid=0 2024-11-10T15:18:27,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742315_1491 (size=12001) 2024-11-10T15:18:27,347 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/d5893eca73d0448d833409464b241a3a 2024-11-10T15:18:27,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/a54f8d83ece44871be0fbbcd5423d4fa as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/a54f8d83ece44871be0fbbcd5423d4fa 2024-11-10T15:18:27,353 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/a54f8d83ece44871be0fbbcd5423d4fa, entries=150, sequenceid=126, filesize=11.7 K 2024-11-10T15:18:27,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/5c6dd6d279c24a359f581e390a0d683a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/5c6dd6d279c24a359f581e390a0d683a 2024-11-10T15:18:27,359 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/5c6dd6d279c24a359f581e390a0d683a, entries=150, sequenceid=126, filesize=11.7 K 2024-11-10T15:18:27,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/d5893eca73d0448d833409464b241a3a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d5893eca73d0448d833409464b241a3a 2024-11-10T15:18:27,364 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d5893eca73d0448d833409464b241a3a, entries=150, sequenceid=126, filesize=11.7 K 2024-11-10T15:18:27,365 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for ba35a6e7578835b556d2567ed9c44ec8 in 929ms, sequenceid=126, compaction requested=false 2024-11-10T15:18:27,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:27,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:27,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-10T15:18:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-10T15:18:27,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-10T15:18:27,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0830 sec 2024-11-10T15:18:27,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.0880 sec 2024-11-10T15:18:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T15:18:27,421 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:27,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-10T15:18:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T15:18:27,423 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:27,424 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:27,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:27,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T15:18:27,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:27,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:27,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:27,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:27,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:27,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:27,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/a70cba5e83964a45b0876c951aaecaf1 is 50, key is test_row_0/A:col10/1731251906831/Put/seqid=0 2024-11-10T15:18:27,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742316_1492 (size=14541) 2024-11-10T15:18:27,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/a70cba5e83964a45b0876c951aaecaf1 2024-11-10T15:18:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T15:18:27,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/b58cd8f3d568449e97fadc0df53cd5e8 is 50, key is test_row_0/B:col10/1731251906831/Put/seqid=0 2024-11-10T15:18:27,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742317_1493 (size=12151) 2024-11-10T15:18:27,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/b58cd8f3d568449e97fadc0df53cd5e8 2024-11-10T15:18:27,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/8a7b168e522c4145b30b62e246fd948d is 50, key is test_row_0/C:col10/1731251906831/Put/seqid=0 2024-11-10T15:18:27,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742318_1494 (size=12151) 2024-11-10T15:18:27,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T15:18:27,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:27,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:27,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:27,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:27,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:27,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251907585, max timestamp=1731251907651Num uniques 859; 2024-11-10T15:18:27,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251907585, max timestamp=1731251907651Num uniques 859; 2024-11-10T15:18:27,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:18:27,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:18:27,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251907585, max timestamp=1731251907651Num uniques 859; 2024-11-10T15:18:27,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:18:27,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T15:18:27,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:27,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:27,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:27,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:27,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T15:18:27,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251967841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:27,851 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_1' in ba35a6e7578835b556d2567ed9c44ec8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:18:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:27,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251967847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:27,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:27,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:27,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:27,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251967852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:27,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251967852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:27,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251967852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:27,865 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in ba35a6e7578835b556d2567ed9c44ec8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:18:27,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:27,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:27,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:27,866 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in ba35a6e7578835b556d2567ed9c44ec8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:18:27,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:27,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:27,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:27,867 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in ba35a6e7578835b556d2567ed9c44ec8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:18:27,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:27,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:27,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:27,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T15:18:27,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:27,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:27,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:27,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:27,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:27,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/8a7b168e522c4145b30b62e246fd948d 2024-11-10T15:18:27,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251967964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:27,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/a70cba5e83964a45b0876c951aaecaf1 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/a70cba5e83964a45b0876c951aaecaf1 2024-11-10T15:18:27,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/a70cba5e83964a45b0876c951aaecaf1, entries=200, sequenceid=140, filesize=14.2 K 2024-11-10T15:18:27,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/b58cd8f3d568449e97fadc0df53cd5e8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/b58cd8f3d568449e97fadc0df53cd5e8 2024-11-10T15:18:27,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/b58cd8f3d568449e97fadc0df53cd5e8, entries=150, sequenceid=140, filesize=11.9 K 2024-11-10T15:18:27,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/8a7b168e522c4145b30b62e246fd948d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/8a7b168e522c4145b30b62e246fd948d 2024-11-10T15:18:27,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/8a7b168e522c4145b30b62e246fd948d, entries=150, sequenceid=140, filesize=11.9 K 2024-11-10T15:18:27,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ba35a6e7578835b556d2567ed9c44ec8 in 489ms, sequenceid=140, compaction requested=true 2024-11-10T15:18:27,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:27,994 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:27,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:27,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:27,994 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:27,994 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:27,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:27,995 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:27,995 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/B is initiating minor compaction (all files) 2024-11-10T15:18:27,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:27,995 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/A is initiating minor compaction (all files) 2024-11-10T15:18:27,995 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/B in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:27,995 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/A in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:27,995 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/247197642bc64224b3384f3cd4045f87, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/5c6dd6d279c24a359f581e390a0d683a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/b58cd8f3d568449e97fadc0df53cd5e8] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=35.5 K 2024-11-10T15:18:27,995 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/0644c7032d9548ce98985ea91ef1dd00, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/a54f8d83ece44871be0fbbcd5423d4fa, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/a70cba5e83964a45b0876c951aaecaf1] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=37.9 K 2024-11-10T15:18:27,995 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 247197642bc64224b3384f3cd4045f87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731251905495 2024-11-10T15:18:27,995 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0644c7032d9548ce98985ea91ef1dd00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731251905495 2024-11-10T15:18:27,996 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a54f8d83ece44871be0fbbcd5423d4fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731251905615 2024-11-10T15:18:27,996 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c6dd6d279c24a359f581e390a0d683a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731251905615 2024-11-10T15:18:27,996 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a70cba5e83964a45b0876c951aaecaf1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731251906764 2024-11-10T15:18:27,996 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting b58cd8f3d568449e97fadc0df53cd5e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731251906810 2024-11-10T15:18:27,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:28,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:28,011 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#A#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:28,012 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/fbbe5a5c814c43fdba4a39d08d2fd3c5 is 50, key is test_row_0/A:col10/1731251906831/Put/seqid=0 2024-11-10T15:18:28,015 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#B#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:28,015 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/f96f746bc7744407993fafc65bc62440 is 50, key is test_row_0/B:col10/1731251906831/Put/seqid=0 2024-11-10T15:18:28,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742319_1495 (size=12493) 2024-11-10T15:18:28,037 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/fbbe5a5c814c43fdba4a39d08d2fd3c5 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fbbe5a5c814c43fdba4a39d08d2fd3c5 2024-11-10T15:18:28,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T15:18:28,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:28,038 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-10T15:18:28,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:28,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:28,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:28,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:28,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:28,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T15:18:28,056 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/A of ba35a6e7578835b556d2567ed9c44ec8 into fbbe5a5c814c43fdba4a39d08d2fd3c5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:28,056 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:28,056 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/A, priority=13, startTime=1731251907993; duration=0sec 2024-11-10T15:18:28,057 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:28,057 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:A 2024-11-10T15:18:28,057 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:28,058 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:28,058 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/C is initiating minor compaction (all files) 2024-11-10T15:18:28,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742320_1496 (size=12493) 2024-11-10T15:18:28,058 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/C in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:28,058 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/f62a066ea8544ba688196205e3f30c3f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d5893eca73d0448d833409464b241a3a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/8a7b168e522c4145b30b62e246fd948d] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=35.5 K 2024-11-10T15:18:28,058 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f62a066ea8544ba688196205e3f30c3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731251905495 2024-11-10T15:18:28,059 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5893eca73d0448d833409464b241a3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731251905615 2024-11-10T15:18:28,059 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a7b168e522c4145b30b62e246fd948d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731251906810 2024-11-10T15:18:28,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/714d8730eb9a4050b3e1dcfdab58fb3a is 50, key is test_row_0/A:col10/1731251907845/Put/seqid=0 2024-11-10T15:18:28,073 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/f96f746bc7744407993fafc65bc62440 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/f96f746bc7744407993fafc65bc62440 2024-11-10T15:18:28,079 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/B of ba35a6e7578835b556d2567ed9c44ec8 into f96f746bc7744407993fafc65bc62440(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:28,079 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:28,079 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/B, priority=13, startTime=1731251907994; duration=0sec 2024-11-10T15:18:28,080 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:28,080 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:B 2024-11-10T15:18:28,088 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#C#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:28,089 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/e9f9d3e39cdb40ebbce65f29188b9011 is 50, key is test_row_0/C:col10/1731251906831/Put/seqid=0 2024-11-10T15:18:28,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742321_1497 (size=12151) 2024-11-10T15:18:28,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742322_1498 (size=12493) 2024-11-10T15:18:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:28,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:28,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251968302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:28,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251968422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:28,528 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/714d8730eb9a4050b3e1dcfdab58fb3a 2024-11-10T15:18:28,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/0baade8f83bc42e3adfe86b30a4cfb4c is 50, key is test_row_0/B:col10/1731251907845/Put/seqid=0 2024-11-10T15:18:28,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742323_1499 (size=12151) 2024-11-10T15:18:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T15:18:28,565 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/0baade8f83bc42e3adfe86b30a4cfb4c 2024-11-10T15:18:28,566 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/e9f9d3e39cdb40ebbce65f29188b9011 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/e9f9d3e39cdb40ebbce65f29188b9011 2024-11-10T15:18:28,573 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/C of ba35a6e7578835b556d2567ed9c44ec8 into e9f9d3e39cdb40ebbce65f29188b9011(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:28,574 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:28,574 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/C, priority=13, startTime=1731251907995; duration=0sec 2024-11-10T15:18:28,574 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:28,574 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:C 2024-11-10T15:18:28,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/78362d9237df4d85a882167ca408eed7 is 50, key is test_row_0/C:col10/1731251907845/Put/seqid=0 2024-11-10T15:18:28,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742324_1500 (size=12151) 2024-11-10T15:18:28,604 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/78362d9237df4d85a882167ca408eed7 2024-11-10T15:18:28,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/714d8730eb9a4050b3e1dcfdab58fb3a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/714d8730eb9a4050b3e1dcfdab58fb3a 2024-11-10T15:18:28,611 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/714d8730eb9a4050b3e1dcfdab58fb3a, entries=150, sequenceid=175, filesize=11.9 K 2024-11-10T15:18:28,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/0baade8f83bc42e3adfe86b30a4cfb4c as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/0baade8f83bc42e3adfe86b30a4cfb4c 2024-11-10T15:18:28,620 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/0baade8f83bc42e3adfe86b30a4cfb4c, entries=150, sequenceid=175, filesize=11.9 K 2024-11-10T15:18:28,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/78362d9237df4d85a882167ca408eed7 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/78362d9237df4d85a882167ca408eed7 2024-11-10T15:18:28,626 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/78362d9237df4d85a882167ca408eed7, entries=150, sequenceid=175, filesize=11.9 K 2024-11-10T15:18:28,627 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for ba35a6e7578835b556d2567ed9c44ec8 in 589ms, sequenceid=175, compaction requested=false 2024-11-10T15:18:28,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:28,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:28,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-10T15:18:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-10T15:18:28,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-10T15:18:28,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2050 sec 2024-11-10T15:18:28,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.2090 sec 2024-11-10T15:18:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:28,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T15:18:28,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:28,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:28,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:28,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:28,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:28,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:28,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/2b5a3b433e864b979ba1dc0dd0f4ee0e is 50, key is test_row_0/A:col10/1731251908300/Put/seqid=0 2024-11-10T15:18:28,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742325_1501 (size=14541) 2024-11-10T15:18:28,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251968836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:28,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251968951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:28,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:28,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:28,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:29,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/2b5a3b433e864b979ba1dc0dd0f4ee0e 2024-11-10T15:18:29,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/29385a7dc5fd43f1a362015c1bd02316 is 50, key is test_row_0/B:col10/1731251908300/Put/seqid=0 2024-11-10T15:18:29,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742326_1502 (size=12151) 2024-11-10T15:18:29,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251969161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:29,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:29,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:29,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/29385a7dc5fd43f1a362015c1bd02316 2024-11-10T15:18:29,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251969481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:29,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/24d1f255d46a4e768143b177f450362e is 50, key is test_row_0/C:col10/1731251908300/Put/seqid=0 2024-11-10T15:18:29,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742327_1503 (size=12151) 2024-11-10T15:18:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T15:18:29,572 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:29,572 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-10T15:18:29,574 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T15:18:29,575 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:29,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T15:18:29,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T15:18:29,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:29,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:29,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:29,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:29,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:29,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251969872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:29,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251969872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:29,880 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_1' in ba35a6e7578835b556d2567ed9c44ec8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:18:29,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:29,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:29,881 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in ba35a6e7578835b556d2567ed9c44ec8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:18:29,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:29,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:29,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:29,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:29,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T15:18:29,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:29,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:29,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:29,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:29,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:29,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251969881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:29,888 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in ba35a6e7578835b556d2567ed9c44ec8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:18:29,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:29,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:29,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T15:18:29,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251969891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:29,906 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in ba35a6e7578835b556d2567ed9c44ec8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4274 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:18:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:29,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/24d1f255d46a4e768143b177f450362e 2024-11-10T15:18:29,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/2b5a3b433e864b979ba1dc0dd0f4ee0e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/2b5a3b433e864b979ba1dc0dd0f4ee0e 2024-11-10T15:18:29,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/2b5a3b433e864b979ba1dc0dd0f4ee0e, entries=200, sequenceid=190, filesize=14.2 K 2024-11-10T15:18:29,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/29385a7dc5fd43f1a362015c1bd02316 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/29385a7dc5fd43f1a362015c1bd02316 2024-11-10T15:18:29,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/29385a7dc5fd43f1a362015c1bd02316, entries=150, sequenceid=190, filesize=11.9 K 2024-11-10T15:18:29,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/24d1f255d46a4e768143b177f450362e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/24d1f255d46a4e768143b177f450362e 2024-11-10T15:18:29,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/24d1f255d46a4e768143b177f450362e, entries=150, sequenceid=190, filesize=11.9 K 2024-11-10T15:18:29,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ba35a6e7578835b556d2567ed9c44ec8 in 1294ms, sequenceid=190, compaction requested=true 2024-11-10T15:18:29,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:29,941 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:29,942 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:29,942 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/A is initiating minor compaction (all files) 2024-11-10T15:18:29,942 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/A in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:29,942 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fbbe5a5c814c43fdba4a39d08d2fd3c5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/714d8730eb9a4050b3e1dcfdab58fb3a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/2b5a3b433e864b979ba1dc0dd0f4ee0e] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=38.3 K 2024-11-10T15:18:29,943 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbbe5a5c814c43fdba4a39d08d2fd3c5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731251906810 2024-11-10T15:18:29,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:29,943 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 714d8730eb9a4050b3e1dcfdab58fb3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731251907777 2024-11-10T15:18:29,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:29,943 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:29,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:29,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:29,943 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b5a3b433e864b979ba1dc0dd0f4ee0e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731251908235 2024-11-10T15:18:29,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:29,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:29,945 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:29,945 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/B is initiating minor compaction (all files) 2024-11-10T15:18:29,945 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/B in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:29,945 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/f96f746bc7744407993fafc65bc62440, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/0baade8f83bc42e3adfe86b30a4cfb4c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/29385a7dc5fd43f1a362015c1bd02316] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=35.9 K 2024-11-10T15:18:29,946 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting f96f746bc7744407993fafc65bc62440, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731251906810 2024-11-10T15:18:29,946 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0baade8f83bc42e3adfe86b30a4cfb4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731251907777 2024-11-10T15:18:29,948 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 29385a7dc5fd43f1a362015c1bd02316, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731251908235 2024-11-10T15:18:29,962 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#A#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:29,962 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/03e96dc1babe4368a83f72aca156bad5 is 50, key is test_row_0/A:col10/1731251908300/Put/seqid=0 2024-11-10T15:18:29,963 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#B#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:29,963 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/5487ad961e6e489a91079c04436ed962 is 50, key is test_row_0/B:col10/1731251908300/Put/seqid=0 2024-11-10T15:18:29,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742328_1504 (size=12595) 2024-11-10T15:18:29,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742329_1505 (size=12595) 2024-11-10T15:18:29,981 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/03e96dc1babe4368a83f72aca156bad5 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/03e96dc1babe4368a83f72aca156bad5 2024-11-10T15:18:29,985 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/A of ba35a6e7578835b556d2567ed9c44ec8 into 03e96dc1babe4368a83f72aca156bad5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:29,985 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:29,985 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/A, priority=13, startTime=1731251909941; duration=0sec 2024-11-10T15:18:29,985 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:29,985 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:A 2024-11-10T15:18:29,985 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:29,988 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:29,988 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/C is initiating minor compaction (all files) 2024-11-10T15:18:29,988 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/C in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:29,988 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/e9f9d3e39cdb40ebbce65f29188b9011, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/78362d9237df4d85a882167ca408eed7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/24d1f255d46a4e768143b177f450362e] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=35.9 K 2024-11-10T15:18:29,989 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9f9d3e39cdb40ebbce65f29188b9011, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731251906810 2024-11-10T15:18:29,989 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78362d9237df4d85a882167ca408eed7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731251907777 2024-11-10T15:18:29,989 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24d1f255d46a4e768143b177f450362e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731251908235 2024-11-10T15:18:29,994 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#C#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:29,995 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/14cbad89502841bcb1d1dabe437f5273 is 50, key is test_row_0/C:col10/1731251908300/Put/seqid=0 2024-11-10T15:18:30,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742330_1506 (size=12595) 2024-11-10T15:18:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:30,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T15:18:30,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251908822, max timestamp=1731251910005Num uniques 859; 2024-11-10T15:18:30,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:30,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251908822, max timestamp=1731251910005Num uniques 859; 2024-11-10T15:18:30,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:18:30,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:30,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:18:30,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:30,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:30,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:30,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:30,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/208f154cfed740fe95b57b6831f05236 is 50, key is test_row_0/A:col10/1731251908822/Put/seqid=0 2024-11-10T15:18:30,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742331_1507 (size=14541) 2024-11-10T15:18:30,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T15:18:30,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:30,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:30,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:30,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T15:18:30,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:30,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:30,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:30,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251910108, max timestamp=1731251910189Num uniques 859; 2024-11-10T15:18:30,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:18:30,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251910108, max timestamp=1731251910189Num uniques 859; 2024-11-10T15:18:30,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:18:30,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251910108, max timestamp=1731251910189Num uniques 859; 2024-11-10T15:18:30,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:18:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T15:18:30,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T15:18:30,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:30,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:30,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:30,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,373 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/5487ad961e6e489a91079c04436ed962 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/5487ad961e6e489a91079c04436ed962 2024-11-10T15:18:30,377 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/B of ba35a6e7578835b556d2567ed9c44ec8 into 5487ad961e6e489a91079c04436ed962(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:30,377 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:30,377 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/B, priority=13, startTime=1731251909943; duration=0sec 2024-11-10T15:18:30,377 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:30,377 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:B 2024-11-10T15:18:30,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251970373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:30,407 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/14cbad89502841bcb1d1dabe437f5273 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/14cbad89502841bcb1d1dabe437f5273 2024-11-10T15:18:30,410 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/C of ba35a6e7578835b556d2567ed9c44ec8 into 14cbad89502841bcb1d1dabe437f5273(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:30,410 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:30,410 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/C, priority=13, startTime=1731251909943; duration=0sec 2024-11-10T15:18:30,410 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:30,410 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:C 2024-11-10T15:18:30,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/208f154cfed740fe95b57b6831f05236 2024-11-10T15:18:30,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/ef695c7501624ecf859e5c7e0ab5f4bf is 50, key is test_row_0/B:col10/1731251908822/Put/seqid=0 2024-11-10T15:18:30,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742332_1508 (size=12151) 2024-11-10T15:18:30,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/ef695c7501624ecf859e5c7e0ab5f4bf 2024-11-10T15:18:30,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/c799de7f03294277b924553dd30f23bc is 50, key is test_row_0/C:col10/1731251908822/Put/seqid=0 2024-11-10T15:18:30,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742333_1509 (size=12151) 2024-11-10T15:18:30,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/c799de7f03294277b924553dd30f23bc 2024-11-10T15:18:30,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/208f154cfed740fe95b57b6831f05236 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/208f154cfed740fe95b57b6831f05236 2024-11-10T15:18:30,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/208f154cfed740fe95b57b6831f05236, entries=200, sequenceid=212, filesize=14.2 K 2024-11-10T15:18:30,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/ef695c7501624ecf859e5c7e0ab5f4bf as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/ef695c7501624ecf859e5c7e0ab5f4bf 2024-11-10T15:18:30,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/ef695c7501624ecf859e5c7e0ab5f4bf, entries=150, sequenceid=212, filesize=11.9 K 2024-11-10T15:18:30,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/c799de7f03294277b924553dd30f23bc as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c799de7f03294277b924553dd30f23bc 2024-11-10T15:18:30,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c799de7f03294277b924553dd30f23bc, entries=150, sequenceid=212, filesize=11.9 K 2024-11-10T15:18:30,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for ba35a6e7578835b556d2567ed9c44ec8 in 451ms, sequenceid=212, compaction requested=false 2024-11-10T15:18:30,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:30,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T15:18:30,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-10T15:18:30,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:30,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:30,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:30,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:30,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:30,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:30,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:30,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:30,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/cb1098cdbdd44e38a71509606dc03e14 is 50, key is test_row_0/A:col10/1731251910323/Put/seqid=0 2024-11-10T15:18:30,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742334_1510 (size=14541) 2024-11-10T15:18:30,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/cb1098cdbdd44e38a71509606dc03e14 2024-11-10T15:18:30,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/ef26de9bd62b4390ac6c89417004cc00 is 50, key is test_row_0/B:col10/1731251910323/Put/seqid=0 2024-11-10T15:18:30,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742335_1511 (size=12151) 2024-11-10T15:18:30,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251970556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:30,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T15:18:30,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:30,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:30,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:30,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251970672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T15:18:30,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T15:18:30,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:30,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:30,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:30,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,884 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T15:18:30,884 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T15:18:30,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251970892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:30,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/ef26de9bd62b4390ac6c89417004cc00 2024-11-10T15:18:30,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/550e63b1d672483fbcff3b5e6ce0aeb2 is 50, key is test_row_0/C:col10/1731251910323/Put/seqid=0 2024-11-10T15:18:30,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742336_1512 (size=12151) 2024-11-10T15:18:30,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T15:18:30,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:30,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:30,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:30,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:31,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T15:18:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:31,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:31,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251971211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:31,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:31,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:31,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:31,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T15:18:31,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:31,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:31,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:31,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:31,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:31,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/550e63b1d672483fbcff3b5e6ce0aeb2 2024-11-10T15:18:31,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/cb1098cdbdd44e38a71509606dc03e14 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/cb1098cdbdd44e38a71509606dc03e14 2024-11-10T15:18:31,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/cb1098cdbdd44e38a71509606dc03e14, entries=200, sequenceid=250, filesize=14.2 K 2024-11-10T15:18:31,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/ef26de9bd62b4390ac6c89417004cc00 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/ef26de9bd62b4390ac6c89417004cc00 2024-11-10T15:18:31,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/ef26de9bd62b4390ac6c89417004cc00, entries=150, sequenceid=250, filesize=11.9 K 2024-11-10T15:18:31,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/550e63b1d672483fbcff3b5e6ce0aeb2 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/550e63b1d672483fbcff3b5e6ce0aeb2 2024-11-10T15:18:31,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/550e63b1d672483fbcff3b5e6ce0aeb2, entries=150, sequenceid=250, filesize=11.9 K 2024-11-10T15:18:31,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for ba35a6e7578835b556d2567ed9c44ec8 in 873ms, sequenceid=250, compaction requested=true 2024-11-10T15:18:31,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:31,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:31,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:31,371 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:31,371 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:31,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:31,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:31,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:31,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:31,372 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:31,372 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:31,373 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/B is initiating minor compaction (all files) 2024-11-10T15:18:31,373 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/A is initiating minor compaction (all files) 2024-11-10T15:18:31,373 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/B in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:31,373 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/A in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:31,373 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/5487ad961e6e489a91079c04436ed962, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/ef695c7501624ecf859e5c7e0ab5f4bf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/ef26de9bd62b4390ac6c89417004cc00] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=36.0 K 2024-11-10T15:18:31,373 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/03e96dc1babe4368a83f72aca156bad5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/208f154cfed740fe95b57b6831f05236, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/cb1098cdbdd44e38a71509606dc03e14] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=40.7 K 2024-11-10T15:18:31,373 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5487ad961e6e489a91079c04436ed962, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731251908235 2024-11-10T15:18:31,373 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03e96dc1babe4368a83f72aca156bad5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731251908235 2024-11-10T15:18:31,373 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting ef695c7501624ecf859e5c7e0ab5f4bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731251908822 2024-11-10T15:18:31,374 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 208f154cfed740fe95b57b6831f05236, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731251908822 2024-11-10T15:18:31,374 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting ef26de9bd62b4390ac6c89417004cc00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731251910323 2024-11-10T15:18:31,374 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb1098cdbdd44e38a71509606dc03e14, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731251910323 2024-11-10T15:18:31,386 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#A#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:31,386 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#B#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:31,386 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/962088b6facb44cc93d36a56a1ae2cfe is 50, key is test_row_0/A:col10/1731251910323/Put/seqid=0 2024-11-10T15:18:31,387 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/dc8efd1fe76c4c5296d66eec977a0842 is 50, key is test_row_0/B:col10/1731251910323/Put/seqid=0 2024-11-10T15:18:31,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742337_1513 (size=12697) 2024-11-10T15:18:31,437 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/962088b6facb44cc93d36a56a1ae2cfe as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/962088b6facb44cc93d36a56a1ae2cfe 2024-11-10T15:18:31,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T15:18:31,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:31,443 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T15:18:31,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:31,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:31,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:31,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:31,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:31,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:31,448 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/A of ba35a6e7578835b556d2567ed9c44ec8 into 962088b6facb44cc93d36a56a1ae2cfe(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:31,449 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:31,449 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/A, priority=13, startTime=1731251911371; duration=0sec 2024-11-10T15:18:31,449 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:31,449 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:A 2024-11-10T15:18:31,449 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:31,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/249c4cf14c70486ca953ee0383f5e7bd is 50, key is test_row_0/A:col10/1731251910555/Put/seqid=0 2024-11-10T15:18:31,451 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:31,451 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/C is initiating minor compaction (all files) 2024-11-10T15:18:31,451 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/C in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:31,451 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/14cbad89502841bcb1d1dabe437f5273, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c799de7f03294277b924553dd30f23bc, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/550e63b1d672483fbcff3b5e6ce0aeb2] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=36.0 K 2024-11-10T15:18:31,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742338_1514 (size=12697) 2024-11-10T15:18:31,456 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14cbad89502841bcb1d1dabe437f5273, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731251908235 2024-11-10T15:18:31,459 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c799de7f03294277b924553dd30f23bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731251908822 2024-11-10T15:18:31,460 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 550e63b1d672483fbcff3b5e6ce0aeb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731251910323 2024-11-10T15:18:31,468 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/dc8efd1fe76c4c5296d66eec977a0842 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/dc8efd1fe76c4c5296d66eec977a0842 2024-11-10T15:18:31,475 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/B of ba35a6e7578835b556d2567ed9c44ec8 into dc8efd1fe76c4c5296d66eec977a0842(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:31,475 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:31,475 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/B, priority=13, startTime=1731251911371; duration=0sec 2024-11-10T15:18:31,475 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:31,475 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:B 2024-11-10T15:18:31,479 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#C#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:31,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742339_1515 (size=12251) 2024-11-10T15:18:31,479 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/8ba7d739e97c46f7834a3fb2f43cb5e7 is 50, key is test_row_0/C:col10/1731251910323/Put/seqid=0 2024-11-10T15:18:31,484 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/249c4cf14c70486ca953ee0383f5e7bd 2024-11-10T15:18:31,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/d95b3128d5214e61b00d4db96cccce71 is 50, key is test_row_0/B:col10/1731251910555/Put/seqid=0 2024-11-10T15:18:31,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742340_1516 (size=12697) 2024-11-10T15:18:31,527 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/8ba7d739e97c46f7834a3fb2f43cb5e7 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/8ba7d739e97c46f7834a3fb2f43cb5e7 2024-11-10T15:18:31,533 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/C of ba35a6e7578835b556d2567ed9c44ec8 into 8ba7d739e97c46f7834a3fb2f43cb5e7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:31,533 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:31,533 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/C, priority=13, startTime=1731251911372; duration=0sec 2024-11-10T15:18:31,533 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:31,533 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:C 2024-11-10T15:18:31,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742341_1517 (size=12251) 2024-11-10T15:18:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T15:18:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:31,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:31,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251911735, max timestamp=1731251911857Num uniques 859; 2024-11-10T15:18:31,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251911735, max timestamp=1731251911857Num uniques 859; 2024-11-10T15:18:31,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:18:31,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251911735, max timestamp=1731251911857Num uniques 859; 2024-11-10T15:18:31,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:18:31,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:18:31,938 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/d95b3128d5214e61b00d4db96cccce71 2024-11-10T15:18:31,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/d0a1af65af0146329f30b64ee4d4f3fb is 50, key is test_row_0/C:col10/1731251910555/Put/seqid=0 2024-11-10T15:18:31,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742342_1518 (size=12251) 2024-11-10T15:18:32,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251971991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:32,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251972111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:32,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:32,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251972321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:32,346 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/d0a1af65af0146329f30b64ee4d4f3fb 2024-11-10T15:18:32,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/249c4cf14c70486ca953ee0383f5e7bd as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/249c4cf14c70486ca953ee0383f5e7bd 2024-11-10T15:18:32,353 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/249c4cf14c70486ca953ee0383f5e7bd, entries=150, sequenceid=260, filesize=12.0 K 2024-11-10T15:18:32,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/d95b3128d5214e61b00d4db96cccce71 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d95b3128d5214e61b00d4db96cccce71 2024-11-10T15:18:32,358 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d95b3128d5214e61b00d4db96cccce71, entries=150, sequenceid=260, filesize=12.0 K 2024-11-10T15:18:32,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/d0a1af65af0146329f30b64ee4d4f3fb as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d0a1af65af0146329f30b64ee4d4f3fb 2024-11-10T15:18:32,362 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d0a1af65af0146329f30b64ee4d4f3fb, entries=150, sequenceid=260, filesize=12.0 K 2024-11-10T15:18:32,363 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ba35a6e7578835b556d2567ed9c44ec8 in 920ms, sequenceid=260, compaction requested=false 2024-11-10T15:18:32,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:32,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:32,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-10T15:18:32,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-10T15:18:32,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-10T15:18:32,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7890 sec 2024-11-10T15:18:32,367 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.7930 sec 2024-11-10T15:18:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:32,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731251911974, max timestamp=1731251912649Num uniques 1018; 2024-11-10T15:18:32,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731251911974, max timestamp=1731251912649Num uniques 1018; 2024-11-10T15:18:32,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T15:18:32,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T15:18:32,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731251911974, max timestamp=1731251912649Num uniques 1018; 2024-11-10T15:18:32,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T15:18:32,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=26.84 KB heapSize=65.39 KB 2024-11-10T15:18:32,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:32,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:32,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:32,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:32,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:32,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:32,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/207aa53a7cc744bfacfb5c0421cb4960 is 50, key is test_row_0/A:col10/1731251912649/Put/seqid=0 2024-11-10T15:18:32,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742343_1519 (size=17181) 2024-11-10T15:18:32,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/207aa53a7cc744bfacfb5c0421cb4960 2024-11-10T15:18:32,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/3fb8fe8ff99148569f835c4905d658c5 is 50, key is test_row_0/B:col10/1731251912649/Put/seqid=0 2024-11-10T15:18:32,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742344_1520 (size=12301) 2024-11-10T15:18:32,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/3fb8fe8ff99148569f835c4905d658c5 2024-11-10T15:18:32,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/c1f1d623a68d4410901ff4e6e414fd84 is 50, key is test_row_0/C:col10/1731251912649/Put/seqid=0 2024-11-10T15:18:32,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742345_1521 (size=12301) 2024-11-10T15:18:32,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251912852, max timestamp=1731251912910Num uniques 859; 2024-11-10T15:18:32,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251912852, max timestamp=1731251912910Num uniques 859; 2024-11-10T15:18:32,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:18:32,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251912852, max timestamp=1731251912910Num uniques 859; 2024-11-10T15:18:32,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:18:32,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:18:33,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731251913103, max timestamp=1731251913132Num uniques 1018; 2024-11-10T15:18:33,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T15:18:33,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731251913103, max timestamp=1731251913132Num uniques 1018; 2024-11-10T15:18:33,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T15:18:33,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731251913103, max timestamp=1731251913132Num uniques 1018; 2024-11-10T15:18:33,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T15:18:33,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/c1f1d623a68d4410901ff4e6e414fd84 2024-11-10T15:18:33,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/207aa53a7cc744bfacfb5c0421cb4960 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/207aa53a7cc744bfacfb5c0421cb4960 2024-11-10T15:18:33,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/207aa53a7cc744bfacfb5c0421cb4960, entries=250, sequenceid=302, filesize=16.8 K 2024-11-10T15:18:33,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/3fb8fe8ff99148569f835c4905d658c5 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/3fb8fe8ff99148569f835c4905d658c5 2024-11-10T15:18:33,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/3fb8fe8ff99148569f835c4905d658c5, entries=150, sequenceid=302, filesize=12.0 K 2024-11-10T15:18:33,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/c1f1d623a68d4410901ff4e6e414fd84 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c1f1d623a68d4410901ff4e6e414fd84 2024-11-10T15:18:33,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c1f1d623a68d4410901ff4e6e414fd84, entries=150, sequenceid=302, filesize=12.0 K 2024-11-10T15:18:33,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=60.38 KB/61830 for ba35a6e7578835b556d2567ed9c44ec8 in 526ms, sequenceid=302, compaction requested=true 2024-11-10T15:18:33,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:33,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:33,181 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:33,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:33,181 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:33,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:33,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:33,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:33,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:33,183 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:33,183 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/A is initiating minor compaction (all files) 2024-11-10T15:18:33,183 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/A in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:33,183 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/962088b6facb44cc93d36a56a1ae2cfe, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/249c4cf14c70486ca953ee0383f5e7bd, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/207aa53a7cc744bfacfb5c0421cb4960] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=41.1 K 2024-11-10T15:18:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:33,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=67.09 KB heapSize=184.92 KB 2024-11-10T15:18:33,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:33,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:33,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:33,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:33,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:33,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:33,185 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:33,186 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/B is initiating minor compaction (all files) 2024-11-10T15:18:33,186 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/B in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:33,186 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/dc8efd1fe76c4c5296d66eec977a0842, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d95b3128d5214e61b00d4db96cccce71, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/3fb8fe8ff99148569f835c4905d658c5] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=36.4 K 2024-11-10T15:18:33,186 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 962088b6facb44cc93d36a56a1ae2cfe, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731251910323 2024-11-10T15:18:33,186 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting dc8efd1fe76c4c5296d66eec977a0842, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731251910323 2024-11-10T15:18:33,186 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 249c4cf14c70486ca953ee0383f5e7bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731251910535 2024-11-10T15:18:33,187 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting d95b3128d5214e61b00d4db96cccce71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731251910535 2024-11-10T15:18:33,187 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 207aa53a7cc744bfacfb5c0421cb4960, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731251911974 2024-11-10T15:18:33,188 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fb8fe8ff99148569f835c4905d658c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731251911974 2024-11-10T15:18:33,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/f25eb1ba1a074d2dbbc0a34feb1c86d1 is 50, key is test_row_0/A:col10/1731251913164/Put/seqid=0 2024-11-10T15:18:33,206 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#B#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:33,206 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/7a5324008f144c8aac3f48b7cdad8926 is 50, key is test_row_0/B:col10/1731251912649/Put/seqid=0 2024-11-10T15:18:33,208 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#A#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:33,208 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/7b0ed8dcc4274318a58980f99978df9e is 50, key is test_row_0/A:col10/1731251912649/Put/seqid=0 2024-11-10T15:18:33,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742346_1522 (size=14741) 2024-11-10T15:18:33,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/f25eb1ba1a074d2dbbc0a34feb1c86d1 2024-11-10T15:18:33,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742347_1523 (size=12949) 2024-11-10T15:18:33,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742348_1524 (size=12949) 2024-11-10T15:18:33,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/2062eb725a7341a699425002a2e60292 is 50, key is test_row_0/B:col10/1731251913164/Put/seqid=0 2024-11-10T15:18:33,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742349_1525 (size=12301) 2024-11-10T15:18:33,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/2062eb725a7341a699425002a2e60292 2024-11-10T15:18:33,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/3147038e25cd413f85e9dffaa6fe1e56 is 50, key is test_row_0/C:col10/1731251913164/Put/seqid=0 2024-11-10T15:18:33,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742350_1526 (size=12301) 2024-11-10T15:18:33,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/3147038e25cd413f85e9dffaa6fe1e56 2024-11-10T15:18:33,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/f25eb1ba1a074d2dbbc0a34feb1c86d1 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/f25eb1ba1a074d2dbbc0a34feb1c86d1 2024-11-10T15:18:33,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/f25eb1ba1a074d2dbbc0a34feb1c86d1, entries=200, sequenceid=346, filesize=14.4 K 2024-11-10T15:18:33,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/2062eb725a7341a699425002a2e60292 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/2062eb725a7341a699425002a2e60292 2024-11-10T15:18:33,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/2062eb725a7341a699425002a2e60292, entries=150, sequenceid=346, filesize=12.0 K 2024-11-10T15:18:33,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/3147038e25cd413f85e9dffaa6fe1e56 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/3147038e25cd413f85e9dffaa6fe1e56 2024-11-10T15:18:33,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/3147038e25cd413f85e9dffaa6fe1e56, entries=150, sequenceid=346, filesize=12.0 K 2024-11-10T15:18:33,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~184.88 KB/189312, currentSize=60.38 KB/61830 for ba35a6e7578835b556d2567ed9c44ec8 in 114ms, sequenceid=346, compaction requested=true 2024-11-10T15:18:33,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:33,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:A, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:33,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T15:18:33,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:B, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:33,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-10T15:18:33,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:33,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-10T15:18:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:33,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T15:18:33,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:33,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:33,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:33,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:33,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:33,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:33,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/cb1892fac9cf44a8b395cf2248eb6dfa is 50, key is test_row_0/A:col10/1731251913301/Put/seqid=0 2024-11-10T15:18:33,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742351_1527 (size=14741) 2024-11-10T15:18:33,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251973496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:33,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251973612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:33,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:33,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:33,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:33,626 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/7a5324008f144c8aac3f48b7cdad8926 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/7a5324008f144c8aac3f48b7cdad8926 2024-11-10T15:18:33,626 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/7b0ed8dcc4274318a58980f99978df9e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/7b0ed8dcc4274318a58980f99978df9e 2024-11-10T15:18:33,633 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/A of ba35a6e7578835b556d2567ed9c44ec8 into 7b0ed8dcc4274318a58980f99978df9e(size=12.6 K), total size for store is 27.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:33,633 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:33,633 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/A, priority=13, startTime=1731251913180; duration=0sec 2024-11-10T15:18:33,633 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-10T15:18:33,633 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:A 2024-11-10T15:18:33,633 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:A 2024-11-10T15:18:33,633 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:18:33,638 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/B of ba35a6e7578835b556d2567ed9c44ec8 into 7a5324008f144c8aac3f48b7cdad8926(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:33,638 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:33,638 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/B, priority=13, startTime=1731251913181; duration=0sec 2024-11-10T15:18:33,638 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-10T15:18:33,638 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:B 2024-11-10T15:18:33,638 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:B 2024-11-10T15:18:33,639 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49550 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:18:33,639 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/C is initiating minor compaction (all files) 2024-11-10T15:18:33,639 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/C in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:33,639 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/8ba7d739e97c46f7834a3fb2f43cb5e7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d0a1af65af0146329f30b64ee4d4f3fb, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c1f1d623a68d4410901ff4e6e414fd84, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/3147038e25cd413f85e9dffaa6fe1e56] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=48.4 K 2024-11-10T15:18:33,639 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-10T15:18:33,639 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T15:18:33,640 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T15:18:33,640 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. because compaction request was cancelled 2024-11-10T15:18:33,640 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:C 2024-11-10T15:18:33,640 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T15:18:33,641 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ba7d739e97c46f7834a3fb2f43cb5e7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731251910323 2024-11-10T15:18:33,642 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T15:18:33,642 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T15:18:33,642 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. because compaction request was cancelled 2024-11-10T15:18:33,642 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:B 2024-11-10T15:18:33,642 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0a1af65af0146329f30b64ee4d4f3fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731251910535 2024-11-10T15:18:33,642 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T15:18:33,643 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1f1d623a68d4410901ff4e6e414fd84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731251911974 2024-11-10T15:18:33,644 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T15:18:33,644 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T15:18:33,644 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. because compaction request was cancelled 2024-11-10T15:18:33,644 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:A 2024-11-10T15:18:33,646 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3147038e25cd413f85e9dffaa6fe1e56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731251913132 2024-11-10T15:18:33,663 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#C#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:33,663 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/c917c2d756ab476fab9a217db998349c is 50, key is test_row_0/C:col10/1731251913164/Put/seqid=0 2024-11-10T15:18:33,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742352_1528 (size=12983) 2024-11-10T15:18:33,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/cb1892fac9cf44a8b395cf2248eb6dfa 2024-11-10T15:18:33,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/14d13e816ffa4c04941371bae8b98517 is 50, key is test_row_0/B:col10/1731251913301/Put/seqid=0 2024-11-10T15:18:33,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742353_1529 (size=12301) 2024-11-10T15:18:33,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/14d13e816ffa4c04941371bae8b98517 2024-11-10T15:18:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T15:18:33,741 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:33,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:33,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-10T15:18:33,744 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T15:18:33,744 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:33,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:33,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/bad08b648f5b42999360d8e430ca9bd9 is 50, key is test_row_0/C:col10/1731251913301/Put/seqid=0 2024-11-10T15:18:33,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742354_1530 (size=12301) 2024-11-10T15:18:33,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/bad08b648f5b42999360d8e430ca9bd9 2024-11-10T15:18:33,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/cb1892fac9cf44a8b395cf2248eb6dfa as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/cb1892fac9cf44a8b395cf2248eb6dfa 2024-11-10T15:18:33,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/cb1892fac9cf44a8b395cf2248eb6dfa, entries=200, sequenceid=359, filesize=14.4 K 2024-11-10T15:18:33,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/14d13e816ffa4c04941371bae8b98517 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/14d13e816ffa4c04941371bae8b98517 2024-11-10T15:18:33,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/14d13e816ffa4c04941371bae8b98517, entries=150, sequenceid=359, filesize=12.0 K 2024-11-10T15:18:33,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/bad08b648f5b42999360d8e430ca9bd9 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/bad08b648f5b42999360d8e430ca9bd9 2024-11-10T15:18:33,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/bad08b648f5b42999360d8e430ca9bd9, entries=150, sequenceid=359, filesize=12.0 K 2024-11-10T15:18:33,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ba35a6e7578835b556d2567ed9c44ec8 in 476ms, sequenceid=359, compaction requested=true 2024-11-10T15:18:33,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:33,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:33,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:33,778 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:33,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:33,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:33,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:33,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T15:18:33,778 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:33,778 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/A is initiating minor compaction (all files) 2024-11-10T15:18:33,778 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/A in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:33,779 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/7b0ed8dcc4274318a58980f99978df9e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/f25eb1ba1a074d2dbbc0a34feb1c86d1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/cb1892fac9cf44a8b395cf2248eb6dfa] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=41.4 K 2024-11-10T15:18:33,779 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b0ed8dcc4274318a58980f99978df9e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731251911974 2024-11-10T15:18:33,779 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting f25eb1ba1a074d2dbbc0a34feb1c86d1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731251913132 2024-11-10T15:18:33,779 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting cb1892fac9cf44a8b395cf2248eb6dfa, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731251913255 2024-11-10T15:18:33,783 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#A#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:33,784 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/5939707e6f7143108693fe4b48a2d1bf is 50, key is test_row_0/A:col10/1731251913301/Put/seqid=0 2024-11-10T15:18:33,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742355_1531 (size=13051) 2024-11-10T15:18:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:33,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T15:18:33,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:33,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:33,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:33,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:33,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:33,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T15:18:33,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/53b8358e3e3a44b59fff76f674e9a947 is 50, key is test_row_0/A:col10/1731251913465/Put/seqid=0 2024-11-10T15:18:33,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742356_1532 (size=14741) 2024-11-10T15:18:33,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/53b8358e3e3a44b59fff76f674e9a947 2024-11-10T15:18:33,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T15:18:33,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:33,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:33,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:33,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:33,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:33,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/392da4a362084ee3a8356c212dbe8ab9 is 50, key is test_row_0/B:col10/1731251913465/Put/seqid=0 2024-11-10T15:18:33,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742357_1533 (size=12301) 2024-11-10T15:18:33,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251973926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:33,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251973927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:33,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251973928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:33,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:33,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:33,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:33,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251973936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:33,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251973937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:34,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251974041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:34,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251974041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:34,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T15:18:34,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:34,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:34,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:34,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:34,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:34,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251974053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:34,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251974053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:34,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T15:18:34,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:34,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251974063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:34,085 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/c917c2d756ab476fab9a217db998349c as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c917c2d756ab476fab9a217db998349c 2024-11-10T15:18:34,094 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/C of ba35a6e7578835b556d2567ed9c44ec8 into c917c2d756ab476fab9a217db998349c(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:34,094 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:34,094 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/C, priority=12, startTime=1731251913298; duration=0sec 2024-11-10T15:18:34,094 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T15:18:34,094 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:C 2024-11-10T15:18:34,094 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:C 2024-11-10T15:18:34,094 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:34,095 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:34,095 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/B is initiating minor compaction (all files) 2024-11-10T15:18:34,095 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/B in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:34,095 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/7a5324008f144c8aac3f48b7cdad8926, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/2062eb725a7341a699425002a2e60292, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/14d13e816ffa4c04941371bae8b98517] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=36.7 K 2024-11-10T15:18:34,095 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a5324008f144c8aac3f48b7cdad8926, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731251911974 2024-11-10T15:18:34,095 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2062eb725a7341a699425002a2e60292, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731251913132 2024-11-10T15:18:34,096 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14d13e816ffa4c04941371bae8b98517, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731251913255 2024-11-10T15:18:34,100 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#B#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:34,101 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/32a70a457f124046b19fe5a48ad5f0e6 is 50, key is test_row_0/B:col10/1731251913301/Put/seqid=0 2024-11-10T15:18:34,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742358_1534 (size=13051) 2024-11-10T15:18:34,199 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/5939707e6f7143108693fe4b48a2d1bf as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/5939707e6f7143108693fe4b48a2d1bf 2024-11-10T15:18:34,202 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/A of ba35a6e7578835b556d2567ed9c44ec8 into 5939707e6f7143108693fe4b48a2d1bf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:34,202 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:34,203 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/A, priority=13, startTime=1731251913778; duration=0sec 2024-11-10T15:18:34,203 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:34,203 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:A 2024-11-10T15:18:34,203 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T15:18:34,203 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T15:18:34,203 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T15:18:34,203 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. because compaction request was cancelled 2024-11-10T15:18:34,203 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:C 2024-11-10T15:18:34,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T15:18:34,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:34,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:34,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:34,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:34,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:34,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:34,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:34,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251974261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:34,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251974261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:34,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:34,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251974271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:34,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251974271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:34,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:34,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251974281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:34,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/392da4a362084ee3a8356c212dbe8ab9 2024-11-10T15:18:34,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/fd9b7aed755649dda828f143799795cf is 50, key is test_row_0/C:col10/1731251913465/Put/seqid=0 2024-11-10T15:18:34,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742359_1535 (size=12301) 2024-11-10T15:18:34,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T15:18:34,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:34,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:34,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:34,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T15:18:34,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T15:18:34,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:34,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:34,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:34,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:34,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:34,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:34,531 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/32a70a457f124046b19fe5a48ad5f0e6 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/32a70a457f124046b19fe5a48ad5f0e6 2024-11-10T15:18:34,535 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/B of ba35a6e7578835b556d2567ed9c44ec8 into 32a70a457f124046b19fe5a48ad5f0e6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:34,535 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:34,535 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/B, priority=13, startTime=1731251913778; duration=0sec 2024-11-10T15:18:34,535 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:34,535 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:B 2024-11-10T15:18:34,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251974581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:34,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:34,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251974581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:34,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:34,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251974591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:34,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:34,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251974592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:34,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:34,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251974601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:34,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T15:18:34,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:34,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:34,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:34,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:34,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:34,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/fd9b7aed755649dda828f143799795cf 2024-11-10T15:18:34,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/53b8358e3e3a44b59fff76f674e9a947 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/53b8358e3e3a44b59fff76f674e9a947 2024-11-10T15:18:34,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/53b8358e3e3a44b59fff76f674e9a947, entries=200, sequenceid=381, filesize=14.4 K 2024-11-10T15:18:34,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/392da4a362084ee3a8356c212dbe8ab9 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/392da4a362084ee3a8356c212dbe8ab9 2024-11-10T15:18:34,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/392da4a362084ee3a8356c212dbe8ab9, entries=150, sequenceid=381, filesize=12.0 K 2024-11-10T15:18:34,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/fd9b7aed755649dda828f143799795cf as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/fd9b7aed755649dda828f143799795cf 2024-11-10T15:18:34,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/fd9b7aed755649dda828f143799795cf, entries=150, sequenceid=381, filesize=12.0 K 2024-11-10T15:18:34,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for ba35a6e7578835b556d2567ed9c44ec8 in 932ms, sequenceid=381, compaction requested=true 2024-11-10T15:18:34,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:34,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:34,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:34,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:34,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T15:18:34,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:34,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-10T15:18:34,769 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:34,769 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T15:18:34,770 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T15:18:34,770 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T15:18:34,770 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. because compaction request was cancelled 2024-11-10T15:18:34,770 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:A 2024-11-10T15:18:34,770 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T15:18:34,771 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:34,771 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/C is initiating minor compaction (all files) 2024-11-10T15:18:34,771 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/C in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:34,771 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c917c2d756ab476fab9a217db998349c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/bad08b648f5b42999360d8e430ca9bd9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/fd9b7aed755649dda828f143799795cf] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=36.7 K 2024-11-10T15:18:34,771 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T15:18:34,771 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T15:18:34,771 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. because compaction request was cancelled 2024-11-10T15:18:34,771 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:B 2024-11-10T15:18:34,771 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c917c2d756ab476fab9a217db998349c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731251913132 2024-11-10T15:18:34,772 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting bad08b648f5b42999360d8e430ca9bd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731251913255 2024-11-10T15:18:34,772 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd9b7aed755649dda828f143799795cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731251913465 2024-11-10T15:18:34,783 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#C#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:34,784 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/72bf646dfd97464e8ff98124fec306b9 is 50, key is test_row_0/C:col10/1731251913465/Put/seqid=0 2024-11-10T15:18:34,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742360_1536 (size=13085) 2024-11-10T15:18:34,808 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/72bf646dfd97464e8ff98124fec306b9 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/72bf646dfd97464e8ff98124fec306b9 2024-11-10T15:18:34,811 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/C of ba35a6e7578835b556d2567ed9c44ec8 into 72bf646dfd97464e8ff98124fec306b9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:34,811 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:34,811 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/C, priority=13, startTime=1731251914769; duration=0sec 2024-11-10T15:18:34,811 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:34,811 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:C 2024-11-10T15:18:34,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T15:18:34,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:34,832 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T15:18:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:34,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/ff19aced93ab45bfb47ca8956f6fa182 is 50, key is test_row_0/A:col10/1731251913934/Put/seqid=0 2024-11-10T15:18:34,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742361_1537 (size=12301) 2024-11-10T15:18:34,850 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/ff19aced93ab45bfb47ca8956f6fa182 2024-11-10T15:18:34,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/24bcbfa90d3b4773b38deef87a8fb291 is 50, key is test_row_0/B:col10/1731251913934/Put/seqid=0 2024-11-10T15:18:34,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742362_1538 (size=12301) 2024-11-10T15:18:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T15:18:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:35,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:35,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251975134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251975135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251975136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251975137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251975141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251975251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251975251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251975251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251975251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,262 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/24bcbfa90d3b4773b38deef87a8fb291 2024-11-10T15:18:35,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251975261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/13e9816f59ce47d4aafa542a1dff10f2 is 50, key is test_row_0/C:col10/1731251913934/Put/seqid=0 2024-11-10T15:18:35,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742363_1539 (size=12301) 2024-11-10T15:18:35,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251975462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251975462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251975462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251975462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251975481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,682 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/13e9816f59ce47d4aafa542a1dff10f2 2024-11-10T15:18:35,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/ff19aced93ab45bfb47ca8956f6fa182 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/ff19aced93ab45bfb47ca8956f6fa182 2024-11-10T15:18:35,706 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/ff19aced93ab45bfb47ca8956f6fa182, entries=150, sequenceid=398, filesize=12.0 K 2024-11-10T15:18:35,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/24bcbfa90d3b4773b38deef87a8fb291 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/24bcbfa90d3b4773b38deef87a8fb291 2024-11-10T15:18:35,713 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/24bcbfa90d3b4773b38deef87a8fb291, entries=150, sequenceid=398, filesize=12.0 K 2024-11-10T15:18:35,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/13e9816f59ce47d4aafa542a1dff10f2 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/13e9816f59ce47d4aafa542a1dff10f2 2024-11-10T15:18:35,719 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/13e9816f59ce47d4aafa542a1dff10f2, entries=150, sequenceid=398, filesize=12.0 K 2024-11-10T15:18:35,720 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ba35a6e7578835b556d2567ed9c44ec8 in 888ms, sequenceid=398, compaction requested=true 2024-11-10T15:18:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-10T15:18:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-10T15:18:35,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-10T15:18:35,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9770 sec 2024-11-10T15:18:35,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.9850 sec 2024-11-10T15:18:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:35,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T15:18:35,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:35,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:35,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:35,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:35,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:35,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:35,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/93d73614ae5b4daf8c1a3d7c9d12b942 is 50, key is test_row_0/A:col10/1731251915140/Put/seqid=0 2024-11-10T15:18:35,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742364_1540 (size=14741) 2024-11-10T15:18:35,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251975817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251975821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251975822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251975825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251975827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T15:18:35,892 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:35,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-10T15:18:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T15:18:35,894 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:35,894 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:35,894 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:35,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251975941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251975941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251975941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251975951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251975951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:35,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:35,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:35,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T15:18:36,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T15:18:36,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:36,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:36,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:36,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251976152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:36,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251976152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:36,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251976152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:36,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251976171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:36,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251976171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:36,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T15:18:36,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:36,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:36,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:36,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T15:18:36,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/93d73614ae5b4daf8c1a3d7c9d12b942 2024-11-10T15:18:36,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/387da5d65bd74cf3a630ff2019fadc58 is 50, key is test_row_0/B:col10/1731251915140/Put/seqid=0 2024-11-10T15:18:36,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742365_1541 (size=12301) 2024-11-10T15:18:36,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T15:18:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:36,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251976471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251976471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:36,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:36,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251976471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:36,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:36,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251976482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:36,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251976491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:36,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T15:18:36,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:36,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:36,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:36,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T15:18:36,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/387da5d65bd74cf3a630ff2019fadc58 2024-11-10T15:18:36,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/2eef1211179e4d799e0b87686a1caf56 is 50, key is test_row_0/C:col10/1731251915140/Put/seqid=0 2024-11-10T15:18:36,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T15:18:36,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:36,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:36,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:36,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742366_1542 (size=12301) 2024-11-10T15:18:36,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T15:18:36,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:36,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:36,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:36,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T15:18:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:36,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:36,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:36,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251976992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:37,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:37,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251976992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:37,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:37,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251976992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:37,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:37,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251977002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:37,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:37,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:37,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:37,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251977011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:37,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T15:18:37,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/2eef1211179e4d799e0b87686a1caf56 2024-11-10T15:18:37,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/93d73614ae5b4daf8c1a3d7c9d12b942 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/93d73614ae5b4daf8c1a3d7c9d12b942 2024-11-10T15:18:37,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/93d73614ae5b4daf8c1a3d7c9d12b942, entries=200, sequenceid=418, filesize=14.4 K 2024-11-10T15:18:37,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/387da5d65bd74cf3a630ff2019fadc58 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/387da5d65bd74cf3a630ff2019fadc58 2024-11-10T15:18:37,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/387da5d65bd74cf3a630ff2019fadc58, entries=150, sequenceid=418, filesize=12.0 K 2024-11-10T15:18:37,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/2eef1211179e4d799e0b87686a1caf56 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/2eef1211179e4d799e0b87686a1caf56 2024-11-10T15:18:37,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/2eef1211179e4d799e0b87686a1caf56, entries=150, sequenceid=418, filesize=12.0 K 2024-11-10T15:18:37,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ba35a6e7578835b556d2567ed9c44ec8 in 1299ms, sequenceid=418, compaction requested=true 2024-11-10T15:18:37,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:37,095 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:18:37,096 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54834 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:18:37,096 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/A is initiating minor compaction (all files) 2024-11-10T15:18:37,096 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/A in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:37,096 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/5939707e6f7143108693fe4b48a2d1bf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/53b8358e3e3a44b59fff76f674e9a947, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/ff19aced93ab45bfb47ca8956f6fa182, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/93d73614ae5b4daf8c1a3d7c9d12b942] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=53.5 K 2024-11-10T15:18:37,096 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5939707e6f7143108693fe4b48a2d1bf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731251913255 2024-11-10T15:18:37,097 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53b8358e3e3a44b59fff76f674e9a947, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731251913465 2024-11-10T15:18:37,097 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff19aced93ab45bfb47ca8956f6fa182, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731251913927 2024-11-10T15:18:37,097 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93d73614ae5b4daf8c1a3d7c9d12b942, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731251915118 2024-11-10T15:18:37,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:37,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:37,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:37,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:37,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:37,103 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:18:37,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:37,104 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:18:37,105 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/B is initiating minor compaction (all files) 2024-11-10T15:18:37,105 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/B in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:37,105 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/32a70a457f124046b19fe5a48ad5f0e6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/392da4a362084ee3a8356c212dbe8ab9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/24bcbfa90d3b4773b38deef87a8fb291, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/387da5d65bd74cf3a630ff2019fadc58] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=48.8 K 2024-11-10T15:18:37,105 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 32a70a457f124046b19fe5a48ad5f0e6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731251913255 2024-11-10T15:18:37,105 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 392da4a362084ee3a8356c212dbe8ab9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731251913465 2024-11-10T15:18:37,105 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 24bcbfa90d3b4773b38deef87a8fb291, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731251913927 2024-11-10T15:18:37,106 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 387da5d65bd74cf3a630ff2019fadc58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731251915118 2024-11-10T15:18:37,114 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#A#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:37,115 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/5570834c8e65474ca6e14a6af5323282 is 50, key is test_row_0/A:col10/1731251915140/Put/seqid=0 2024-11-10T15:18:37,118 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#B#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:37,118 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/03ed8563dc8a4734bd7687300eae603d is 50, key is test_row_0/B:col10/1731251915140/Put/seqid=0 2024-11-10T15:18:37,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T15:18:37,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:37,125 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T15:18:37,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:37,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:37,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:37,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:37,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:37,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:37,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742367_1543 (size=13187) 2024-11-10T15:18:37,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742368_1544 (size=13187) 2024-11-10T15:18:37,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/67853175184b416c884cf927c8b4b86a is 50, key is test_row_0/A:col10/1731251915797/Put/seqid=0 2024-11-10T15:18:37,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742369_1545 (size=12301) 2024-11-10T15:18:37,152 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/67853175184b416c884cf927c8b4b86a 2024-11-10T15:18:37,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/420fb05a7c254fb5b5c1e151d31196bc is 50, key is test_row_0/B:col10/1731251915797/Put/seqid=0 2024-11-10T15:18:37,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742370_1546 (size=12301) 2024-11-10T15:18:37,161 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/420fb05a7c254fb5b5c1e151d31196bc 2024-11-10T15:18:37,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/d759cd192e3a44e9a2de9a1d0b51b5d5 is 50, key is test_row_0/C:col10/1731251915797/Put/seqid=0 2024-11-10T15:18:37,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742371_1547 (size=12301) 2024-11-10T15:18:37,537 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/5570834c8e65474ca6e14a6af5323282 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/5570834c8e65474ca6e14a6af5323282 2024-11-10T15:18:37,541 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/A of ba35a6e7578835b556d2567ed9c44ec8 into 5570834c8e65474ca6e14a6af5323282(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:37,541 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:37,541 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/A, priority=12, startTime=1731251917095; duration=0sec 2024-11-10T15:18:37,541 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:37,541 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:A 2024-11-10T15:18:37,541 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:37,542 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/03ed8563dc8a4734bd7687300eae603d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/03ed8563dc8a4734bd7687300eae603d 2024-11-10T15:18:37,547 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/B of ba35a6e7578835b556d2567ed9c44ec8 into 03ed8563dc8a4734bd7687300eae603d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:37,547 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:37,547 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/B, priority=12, startTime=1731251917103; duration=0sec 2024-11-10T15:18:37,547 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:37,547 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:B 2024-11-10T15:18:37,559 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:37,559 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/C is initiating minor compaction (all files) 2024-11-10T15:18:37,559 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/C in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:37,559 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/72bf646dfd97464e8ff98124fec306b9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/13e9816f59ce47d4aafa542a1dff10f2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/2eef1211179e4d799e0b87686a1caf56] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=36.8 K 2024-11-10T15:18:37,559 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72bf646dfd97464e8ff98124fec306b9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731251913465 2024-11-10T15:18:37,560 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13e9816f59ce47d4aafa542a1dff10f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731251913927 2024-11-10T15:18:37,560 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eef1211179e4d799e0b87686a1caf56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731251915118 2024-11-10T15:18:37,575 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#C#compaction#461 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:37,576 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/0ed78e74a06c4336ab84cd57c569bc38 is 50, key is test_row_0/C:col10/1731251915140/Put/seqid=0 2024-11-10T15:18:37,578 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/d759cd192e3a44e9a2de9a1d0b51b5d5 2024-11-10T15:18:37,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/67853175184b416c884cf927c8b4b86a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/67853175184b416c884cf927c8b4b86a 2024-11-10T15:18:37,597 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/67853175184b416c884cf927c8b4b86a, entries=150, sequenceid=430, filesize=12.0 K 2024-11-10T15:18:37,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/420fb05a7c254fb5b5c1e151d31196bc as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/420fb05a7c254fb5b5c1e151d31196bc 2024-11-10T15:18:37,604 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/420fb05a7c254fb5b5c1e151d31196bc, entries=150, sequenceid=430, filesize=12.0 K 2024-11-10T15:18:37,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/d759cd192e3a44e9a2de9a1d0b51b5d5 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d759cd192e3a44e9a2de9a1d0b51b5d5 2024-11-10T15:18:37,610 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d759cd192e3a44e9a2de9a1d0b51b5d5, entries=150, sequenceid=430, filesize=12.0 K 2024-11-10T15:18:37,611 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for ba35a6e7578835b556d2567ed9c44ec8 in 485ms, sequenceid=430, compaction requested=false 2024-11-10T15:18:37,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:37,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:37,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-10T15:18:37,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-10T15:18:37,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-10T15:18:37,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7190 sec 2024-11-10T15:18:37,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.7240 sec 2024-11-10T15:18:37,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742372_1548 (size=13187) 2024-11-10T15:18:37,630 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/0ed78e74a06c4336ab84cd57c569bc38 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0ed78e74a06c4336ab84cd57c569bc38 2024-11-10T15:18:37,638 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/C of ba35a6e7578835b556d2567ed9c44ec8 into 0ed78e74a06c4336ab84cd57c569bc38(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:37,638 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:37,638 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/C, priority=13, startTime=1731251917103; duration=0sec 2024-11-10T15:18:37,638 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:37,638 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:C 2024-11-10T15:18:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:38,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T15:18:38,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:38,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:38,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:38,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:38,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:38,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:38,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/692a5d7b96f140ffbfd2d74936a453fc is 50, key is test_row_0/A:col10/1731251918011/Put/seqid=0 2024-11-10T15:18:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T15:18:38,042 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:38,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-10T15:18:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T15:18:38,047 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:38,048 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:38,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:38,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742373_1549 (size=12301) 2024-11-10T15:18:38,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731251918058, max timestamp=1731251918082Num uniques 859; 2024-11-10T15:18:38,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731251918058, max timestamp=1731251918082Num uniques 859; 2024-11-10T15:18:38,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T15:18:38,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T15:18:38,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731251918058, max timestamp=1731251918082Num uniques 859; 2024-11-10T15:18:38,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T15:18:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T15:18:38,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251978128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251978130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251978155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251978155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251978160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T15:18:38,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:38,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:38,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:38,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:38,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:38,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:38,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251978262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251978262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251978281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251978281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251978281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T15:18:38,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:38,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:38,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:38,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:38,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T15:18:38,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/692a5d7b96f140ffbfd2d74936a453fc 2024-11-10T15:18:38,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/4a7d9862984740889bf42e516cb141d7 is 50, key is test_row_0/B:col10/1731251918011/Put/seqid=0 2024-11-10T15:18:38,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742374_1550 (size=12301) 2024-11-10T15:18:38,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251978482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251978482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251978502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T15:18:38,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:38,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:38,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251978502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:38,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:38,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251978502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:38,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T15:18:38,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:38,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:38,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:38,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:38,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T15:18:38,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251978801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251978802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T15:18:38,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:38,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:38,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:38,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:38,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:38,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251978821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251978821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251978821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:38,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/4a7d9862984740889bf42e516cb141d7 2024-11-10T15:18:38,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/f4557191085343368b455835bf4e7479 is 50, key is test_row_0/C:col10/1731251918011/Put/seqid=0 2024-11-10T15:18:38,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742375_1551 (size=12301) 2024-11-10T15:18:38,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/f4557191085343368b455835bf4e7479 2024-11-10T15:18:38,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/692a5d7b96f140ffbfd2d74936a453fc as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/692a5d7b96f140ffbfd2d74936a453fc 2024-11-10T15:18:38,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/692a5d7b96f140ffbfd2d74936a453fc, entries=150, sequenceid=443, filesize=12.0 K 2024-11-10T15:18:38,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/4a7d9862984740889bf42e516cb141d7 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/4a7d9862984740889bf42e516cb141d7 2024-11-10T15:18:38,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/4a7d9862984740889bf42e516cb141d7, entries=150, sequenceid=443, filesize=12.0 K 2024-11-10T15:18:38,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/f4557191085343368b455835bf4e7479 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/f4557191085343368b455835bf4e7479 2024-11-10T15:18:38,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/f4557191085343368b455835bf4e7479, entries=150, sequenceid=443, filesize=12.0 K 2024-11-10T15:18:38,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for ba35a6e7578835b556d2567ed9c44ec8 in 912ms, sequenceid=443, compaction requested=true 2024-11-10T15:18:38,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:38,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:38,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:38,936 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:38,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:38,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:38,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:38,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T15:18:38,937 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:38,937 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:38,938 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/A is initiating minor compaction (all files) 2024-11-10T15:18:38,938 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/A in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:38,938 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/5570834c8e65474ca6e14a6af5323282, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/67853175184b416c884cf927c8b4b86a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/692a5d7b96f140ffbfd2d74936a453fc] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=36.9 K 2024-11-10T15:18:38,938 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5570834c8e65474ca6e14a6af5323282, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731251915118 2024-11-10T15:18:38,938 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:38,938 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/B is initiating minor compaction (all files) 2024-11-10T15:18:38,938 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/B in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:38,938 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67853175184b416c884cf927c8b4b86a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731251915797 2024-11-10T15:18:38,938 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/03ed8563dc8a4734bd7687300eae603d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/420fb05a7c254fb5b5c1e151d31196bc, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/4a7d9862984740889bf42e516cb141d7] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=36.9 K 2024-11-10T15:18:38,938 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 692a5d7b96f140ffbfd2d74936a453fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731251918011 2024-11-10T15:18:38,939 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 03ed8563dc8a4734bd7687300eae603d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731251915118 2024-11-10T15:18:38,939 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 420fb05a7c254fb5b5c1e151d31196bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731251915797 2024-11-10T15:18:38,939 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a7d9862984740889bf42e516cb141d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731251918011 2024-11-10T15:18:38,945 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#B#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:38,945 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/f9a03fbf3a3c4aaf90d845e0a65f1f2f is 50, key is test_row_0/B:col10/1731251918011/Put/seqid=0 2024-11-10T15:18:38,946 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#A#compaction#466 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:38,946 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/fb1276fcf71f44a787068dab86b460d1 is 50, key is test_row_0/A:col10/1731251918011/Put/seqid=0 2024-11-10T15:18:38,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742376_1552 (size=13289) 2024-11-10T15:18:38,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742377_1553 (size=13289) 2024-11-10T15:18:38,954 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/fb1276fcf71f44a787068dab86b460d1 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fb1276fcf71f44a787068dab86b460d1 2024-11-10T15:18:38,955 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/f9a03fbf3a3c4aaf90d845e0a65f1f2f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/f9a03fbf3a3c4aaf90d845e0a65f1f2f 2024-11-10T15:18:38,958 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/A of ba35a6e7578835b556d2567ed9c44ec8 into fb1276fcf71f44a787068dab86b460d1(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:38,958 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:38,958 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/A, priority=13, startTime=1731251918936; duration=0sec 2024-11-10T15:18:38,958 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:38,958 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:A 2024-11-10T15:18:38,958 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:38,959 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/B of ba35a6e7578835b556d2567ed9c44ec8 into f9a03fbf3a3c4aaf90d845e0a65f1f2f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:38,959 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:38,959 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:38,959 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/B, priority=13, startTime=1731251918936; duration=0sec 2024-11-10T15:18:38,959 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/C is initiating minor compaction (all files) 2024-11-10T15:18:38,959 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:38,959 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:B 2024-11-10T15:18:38,959 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/C in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:38,959 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0ed78e74a06c4336ab84cd57c569bc38, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d759cd192e3a44e9a2de9a1d0b51b5d5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/f4557191085343368b455835bf4e7479] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=36.9 K 2024-11-10T15:18:38,959 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ed78e74a06c4336ab84cd57c569bc38, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731251915118 2024-11-10T15:18:38,960 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d759cd192e3a44e9a2de9a1d0b51b5d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731251915797 2024-11-10T15:18:38,960 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4557191085343368b455835bf4e7479, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731251918011 2024-11-10T15:18:38,965 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#C#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:38,966 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/ae499559a9bb491d8ca12868309c4141 is 50, key is test_row_0/C:col10/1731251918011/Put/seqid=0 2024-11-10T15:18:38,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T15:18:38,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:38,969 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-10T15:18:38,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:38,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:38,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:38,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:38,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:38,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:38,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/467ffc41c9fa4a6484660d73b4f4ed97 is 50, key is test_row_0/A:col10/1731251918123/Put/seqid=0 2024-11-10T15:18:38,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742378_1554 (size=13289) 2024-11-10T15:18:39,000 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/ae499559a9bb491d8ca12868309c4141 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/ae499559a9bb491d8ca12868309c4141 2024-11-10T15:18:39,004 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/C of ba35a6e7578835b556d2567ed9c44ec8 into ae499559a9bb491d8ca12868309c4141(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:39,004 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:39,004 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/C, priority=13, startTime=1731251918936; duration=0sec 2024-11-10T15:18:39,005 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:39,005 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:C 2024-11-10T15:18:39,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742379_1555 (size=12301) 2024-11-10T15:18:39,014 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/467ffc41c9fa4a6484660d73b4f4ed97 2024-11-10T15:18:39,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/3bf5403086b84bf397df177d4352d042 is 50, key is test_row_0/B:col10/1731251918123/Put/seqid=0 2024-11-10T15:18:39,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742380_1556 (size=12301) 2024-11-10T15:18:39,062 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/3bf5403086b84bf397df177d4352d042 2024-11-10T15:18:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/299a9444ed2847a6bb7fa30cdc820142 is 50, key is test_row_0/C:col10/1731251918123/Put/seqid=0 2024-11-10T15:18:39,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742381_1557 (size=12301) 2024-11-10T15:18:39,089 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/299a9444ed2847a6bb7fa30cdc820142 2024-11-10T15:18:39,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/467ffc41c9fa4a6484660d73b4f4ed97 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/467ffc41c9fa4a6484660d73b4f4ed97 2024-11-10T15:18:39,102 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/467ffc41c9fa4a6484660d73b4f4ed97, entries=150, sequenceid=481, filesize=12.0 K 2024-11-10T15:18:39,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/3bf5403086b84bf397df177d4352d042 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/3bf5403086b84bf397df177d4352d042 2024-11-10T15:18:39,109 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/3bf5403086b84bf397df177d4352d042, entries=150, sequenceid=481, filesize=12.0 K 2024-11-10T15:18:39,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/299a9444ed2847a6bb7fa30cdc820142 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/299a9444ed2847a6bb7fa30cdc820142 2024-11-10T15:18:39,114 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/299a9444ed2847a6bb7fa30cdc820142, entries=150, sequenceid=481, filesize=12.0 K 2024-11-10T15:18:39,114 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=0 B/0 for ba35a6e7578835b556d2567ed9c44ec8 in 145ms, sequenceid=481, compaction requested=false 2024-11-10T15:18:39,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:39,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:39,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-10T15:18:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-10T15:18:39,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-10T15:18:39,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0690 sec 2024-11-10T15:18:39,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.0780 sec 2024-11-10T15:18:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T15:18:39,181 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:39,182 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-10T15:18:39,184 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T15:18:39,184 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:39,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T15:18:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:39,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T15:18:39,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:39,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:39,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:39,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:39,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:39,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:39,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/6656d7882c094064b9c0ae86318a7e6b is 50, key is test_row_0/A:col10/1731251919321/Put/seqid=0 2024-11-10T15:18:39,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T15:18:39,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:39,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:39,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:39,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:39,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:39,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742382_1558 (size=17181) 2024-11-10T15:18:39,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731251919361, max timestamp=1731251919389Num uniques 859; 2024-11-10T15:18:39,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-10T15:18:39,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731251919361, max timestamp=1731251919389Num uniques 859; 2024-11-10T15:18:39,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-10T15:18:39,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731251919361, max timestamp=1731251919389Num uniques 859; 2024-11-10T15:18:39,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-10T15:18:39,425 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T15:18:39,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251979430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:39,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251979431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:39,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251979433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:39,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251979451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:39,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251979453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:39,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T15:18:39,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:39,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:39,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:39,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:39,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T15:18:39,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251979562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:39,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251979562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251979563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251979563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:39,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251979581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:39,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:39,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T15:18:39,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:39,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:39,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:39,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:39,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:39,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/6656d7882c094064b9c0ae86318a7e6b 2024-11-10T15:18:39,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/be35670bf5ee412baff2356eda5fdbe2 is 50, key is test_row_0/B:col10/1731251919321/Put/seqid=0 2024-11-10T15:18:39,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742383_1559 (size=12301) 2024-11-10T15:18:39,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/be35670bf5ee412baff2356eda5fdbe2 2024-11-10T15:18:39,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/045b671cbce34734a4ec90d86ac28548 is 50, key is test_row_0/C:col10/1731251919321/Put/seqid=0 2024-11-10T15:18:39,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742384_1560 (size=12301) 2024-11-10T15:18:39,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251979781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:39,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251979781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251979781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:39,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251979781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:39,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T15:18:39,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:39,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:39,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:39,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:39,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T15:18:39,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251979801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:39,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:39,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:39,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T15:18:39,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:39,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:39,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:39,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:39,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:40,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251980101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251980101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251980101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251980102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T15:18:40,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:40,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:40,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:40,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:40,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:40,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251980121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/045b671cbce34734a4ec90d86ac28548 2024-11-10T15:18:40,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/6656d7882c094064b9c0ae86318a7e6b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/6656d7882c094064b9c0ae86318a7e6b 2024-11-10T15:18:40,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/6656d7882c094064b9c0ae86318a7e6b, entries=250, sequenceid=492, filesize=16.8 K 2024-11-10T15:18:40,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/be35670bf5ee412baff2356eda5fdbe2 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/be35670bf5ee412baff2356eda5fdbe2 2024-11-10T15:18:40,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/be35670bf5ee412baff2356eda5fdbe2, entries=150, sequenceid=492, filesize=12.0 K 2024-11-10T15:18:40,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/045b671cbce34734a4ec90d86ac28548 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/045b671cbce34734a4ec90d86ac28548 2024-11-10T15:18:40,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/045b671cbce34734a4ec90d86ac28548, entries=150, sequenceid=492, filesize=12.0 K 2024-11-10T15:18:40,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for ba35a6e7578835b556d2567ed9c44ec8 in 861ms, sequenceid=492, compaction requested=true 2024-11-10T15:18:40,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:40,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:40,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:40,186 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:40,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:40,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:40,186 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:40,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:40,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:40,187 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:40,187 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/A is initiating minor compaction (all files) 2024-11-10T15:18:40,187 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/A in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:40,187 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fb1276fcf71f44a787068dab86b460d1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/467ffc41c9fa4a6484660d73b4f4ed97, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/6656d7882c094064b9c0ae86318a7e6b] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=41.8 K 2024-11-10T15:18:40,187 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:40,187 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/B is initiating minor compaction (all files) 2024-11-10T15:18:40,187 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/B in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:40,187 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/f9a03fbf3a3c4aaf90d845e0a65f1f2f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/3bf5403086b84bf397df177d4352d042, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/be35670bf5ee412baff2356eda5fdbe2] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=37.0 K 2024-11-10T15:18:40,187 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb1276fcf71f44a787068dab86b460d1, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731251918011 2024-11-10T15:18:40,188 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting f9a03fbf3a3c4aaf90d845e0a65f1f2f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731251918011 2024-11-10T15:18:40,188 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 467ffc41c9fa4a6484660d73b4f4ed97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1731251918123 2024-11-10T15:18:40,188 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bf5403086b84bf397df177d4352d042, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1731251918123 2024-11-10T15:18:40,189 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting be35670bf5ee412baff2356eda5fdbe2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731251919320 2024-11-10T15:18:40,189 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6656d7882c094064b9c0ae86318a7e6b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731251919314 2024-11-10T15:18:40,195 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#A#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:40,196 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/7cda6e0d05fe45be85914ae3f9976ffb is 50, key is test_row_0/A:col10/1731251919321/Put/seqid=0 2024-11-10T15:18:40,198 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#B#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:40,198 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/f05a5e013ca84911bbcf2a2758df93de is 50, key is test_row_0/B:col10/1731251919321/Put/seqid=0 2024-11-10T15:18:40,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742386_1562 (size=13391) 2024-11-10T15:18:40,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742385_1561 (size=13391) 2024-11-10T15:18:40,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T15:18:40,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:40,263 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-10T15:18:40,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:40,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:40,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:40,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:40,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:40,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:40,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/bfa6c298a8e546089943dffe1d21ae19 is 50, key is test_row_0/A:col10/1731251919431/Put/seqid=0 2024-11-10T15:18:40,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742387_1563 (size=12301) 2024-11-10T15:18:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T15:18:40,607 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/f05a5e013ca84911bbcf2a2758df93de as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/f05a5e013ca84911bbcf2a2758df93de 2024-11-10T15:18:40,609 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/7cda6e0d05fe45be85914ae3f9976ffb as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/7cda6e0d05fe45be85914ae3f9976ffb 2024-11-10T15:18:40,612 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/B of ba35a6e7578835b556d2567ed9c44ec8 into f05a5e013ca84911bbcf2a2758df93de(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:40,613 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:40,613 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/B, priority=13, startTime=1731251920186; duration=0sec 2024-11-10T15:18:40,613 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:40,613 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:B 2024-11-10T15:18:40,613 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:40,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:40,615 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:40,615 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/C is initiating minor compaction (all files) 2024-11-10T15:18:40,615 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/C in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:40,616 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/ae499559a9bb491d8ca12868309c4141, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/299a9444ed2847a6bb7fa30cdc820142, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/045b671cbce34734a4ec90d86ac28548] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=37.0 K 2024-11-10T15:18:40,616 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting ae499559a9bb491d8ca12868309c4141, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731251918011 2024-11-10T15:18:40,616 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 299a9444ed2847a6bb7fa30cdc820142, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1731251918123 2024-11-10T15:18:40,617 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/A of ba35a6e7578835b556d2567ed9c44ec8 into 7cda6e0d05fe45be85914ae3f9976ffb(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:40,617 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:40,617 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/A, priority=13, startTime=1731251920186; duration=0sec 2024-11-10T15:18:40,617 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:40,617 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:A 2024-11-10T15:18:40,617 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 045b671cbce34734a4ec90d86ac28548, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731251919320 2024-11-10T15:18:40,624 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#C#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:40,624 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/2abf30c9f25c46f9a6045ecba6b1af65 is 50, key is test_row_0/C:col10/1731251919321/Put/seqid=0 2024-11-10T15:18:40,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742388_1564 (size=13391) 2024-11-10T15:18:40,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251980631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251980638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251980639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251980640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251980639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,675 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/bfa6c298a8e546089943dffe1d21ae19 2024-11-10T15:18:40,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/eb163e3147174d36932232bddf28e390 is 50, key is test_row_0/B:col10/1731251919431/Put/seqid=0 2024-11-10T15:18:40,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742389_1565 (size=12301) 2024-11-10T15:18:40,701 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/eb163e3147174d36932232bddf28e390 2024-11-10T15:18:40,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/0f6a5116dd2f4f69949d86654542a570 is 50, key is test_row_0/C:col10/1731251919431/Put/seqid=0 2024-11-10T15:18:40,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742390_1566 (size=12301) 2024-11-10T15:18:40,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251980761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251980762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251980762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251980762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251980981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251980981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251980982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251980982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:41,037 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/2abf30c9f25c46f9a6045ecba6b1af65 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/2abf30c9f25c46f9a6045ecba6b1af65 2024-11-10T15:18:41,041 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/C of ba35a6e7578835b556d2567ed9c44ec8 into 2abf30c9f25c46f9a6045ecba6b1af65(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:41,041 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:41,041 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/C, priority=13, startTime=1731251920186; duration=0sec 2024-11-10T15:18:41,041 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:41,041 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:C 2024-11-10T15:18:41,126 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/0f6a5116dd2f4f69949d86654542a570 2024-11-10T15:18:41,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/bfa6c298a8e546089943dffe1d21ae19 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/bfa6c298a8e546089943dffe1d21ae19 2024-11-10T15:18:41,132 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/bfa6c298a8e546089943dffe1d21ae19, entries=150, sequenceid=527, filesize=12.0 K 2024-11-10T15:18:41,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/eb163e3147174d36932232bddf28e390 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/eb163e3147174d36932232bddf28e390 2024-11-10T15:18:41,136 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/eb163e3147174d36932232bddf28e390, entries=150, sequenceid=527, filesize=12.0 K 2024-11-10T15:18:41,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/0f6a5116dd2f4f69949d86654542a570 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0f6a5116dd2f4f69949d86654542a570 2024-11-10T15:18:41,139 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0f6a5116dd2f4f69949d86654542a570, entries=150, sequenceid=527, filesize=12.0 K 2024-11-10T15:18:41,139 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=46.96 KB/48090 for ba35a6e7578835b556d2567ed9c44ec8 in 876ms, sequenceid=527, compaction requested=false 2024-11-10T15:18:41,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:41,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:41,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-10T15:18:41,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-10T15:18:41,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-10T15:18:41,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9560 sec 2024-11-10T15:18:41,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.9600 sec 2024-11-10T15:18:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:41,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T15:18:41,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:41,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:41,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:41,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:41,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:41,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:41,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/937aaea9d0114b84b2f517821ba7c135 is 50, key is test_row_0/A:col10/1731251921303/Put/seqid=0 2024-11-10T15:18:41,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742391_1567 (size=14741) 2024-11-10T15:18:41,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/937aaea9d0114b84b2f517821ba7c135 2024-11-10T15:18:41,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/d8248d1b2bc74b398b7e0ad66071709e is 50, key is test_row_0/B:col10/1731251921303/Put/seqid=0 2024-11-10T15:18:41,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742392_1568 (size=12301) 2024-11-10T15:18:41,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/d8248d1b2bc74b398b7e0ad66071709e 2024-11-10T15:18:41,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/18e8c02b336144b7aa99449b0b8bbb58 is 50, key is test_row_0/C:col10/1731251921303/Put/seqid=0 2024-11-10T15:18:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T15:18:41,332 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:41,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-10T15:18:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T15:18:41,333 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:41,334 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:41,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:41,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742393_1569 (size=12301) 2024-11-10T15:18:41,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/18e8c02b336144b7aa99449b0b8bbb58 2024-11-10T15:18:41,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/937aaea9d0114b84b2f517821ba7c135 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/937aaea9d0114b84b2f517821ba7c135 2024-11-10T15:18:41,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/937aaea9d0114b84b2f517821ba7c135, entries=200, sequenceid=541, filesize=14.4 K 2024-11-10T15:18:41,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/d8248d1b2bc74b398b7e0ad66071709e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d8248d1b2bc74b398b7e0ad66071709e 2024-11-10T15:18:41,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d8248d1b2bc74b398b7e0ad66071709e, entries=150, sequenceid=541, filesize=12.0 K 2024-11-10T15:18:41,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/18e8c02b336144b7aa99449b0b8bbb58 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/18e8c02b336144b7aa99449b0b8bbb58 2024-11-10T15:18:41,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/18e8c02b336144b7aa99449b0b8bbb58, entries=150, sequenceid=541, filesize=12.0 K 2024-11-10T15:18:41,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=100.63 KB/103050 for ba35a6e7578835b556d2567ed9c44ec8 in 48ms, sequenceid=541, compaction requested=true 2024-11-10T15:18:41,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:41,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:41,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:41,352 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:41,352 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:41,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:41,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:41,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba35a6e7578835b556d2567ed9c44ec8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:41,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:41,356 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:41,356 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:41,356 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/B is initiating minor compaction (all files) 2024-11-10T15:18:41,356 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/A is initiating minor compaction (all files) 2024-11-10T15:18:41,356 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/A in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:41,356 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/B in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:41,356 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/7cda6e0d05fe45be85914ae3f9976ffb, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/bfa6c298a8e546089943dffe1d21ae19, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/937aaea9d0114b84b2f517821ba7c135] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=39.5 K 2024-11-10T15:18:41,356 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/f05a5e013ca84911bbcf2a2758df93de, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/eb163e3147174d36932232bddf28e390, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d8248d1b2bc74b398b7e0ad66071709e] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=37.1 K 2024-11-10T15:18:41,357 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting f05a5e013ca84911bbcf2a2758df93de, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731251919320 2024-11-10T15:18:41,357 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cda6e0d05fe45be85914ae3f9976ffb, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731251919320 2024-11-10T15:18:41,357 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfa6c298a8e546089943dffe1d21ae19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731251919429 2024-11-10T15:18:41,359 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting eb163e3147174d36932232bddf28e390, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731251919429 2024-11-10T15:18:41,360 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting d8248d1b2bc74b398b7e0ad66071709e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1731251920635 2024-11-10T15:18:41,360 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 937aaea9d0114b84b2f517821ba7c135, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1731251920635 2024-11-10T15:18:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:41,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T15:18:41,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:41,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:41,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:41,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:41,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:41,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:41,366 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#A#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:41,366 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/fb9a8130b54340eb8404d6fb63f93745 is 50, key is test_row_0/A:col10/1731251921303/Put/seqid=0 2024-11-10T15:18:41,370 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#B#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:41,371 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/2f82ed07e7a148a684809a1e7f8f0610 is 50, key is test_row_0/B:col10/1731251921303/Put/seqid=0 2024-11-10T15:18:41,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/909bda7313a04e5d821a5242c0e94087 is 50, key is test_row_0/A:col10/1731251921326/Put/seqid=0 2024-11-10T15:18:41,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742394_1570 (size=13493) 2024-11-10T15:18:41,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251981380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:41,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251981388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:41,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742395_1571 (size=13493) 2024-11-10T15:18:41,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742396_1572 (size=12301) 2024-11-10T15:18:41,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/909bda7313a04e5d821a5242c0e94087 2024-11-10T15:18:41,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251981394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:41,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251981395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:41,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/cde6ed3e4eec4fbdbd98998d63e5d8fc is 50, key is test_row_0/B:col10/1731251921326/Put/seqid=0 2024-11-10T15:18:41,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742397_1573 (size=12301) 2024-11-10T15:18:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T15:18:41,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T15:18:41,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:41,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:41,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:41,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:41,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:41,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251981501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:41,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:41,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251981501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:41,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:41,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:41,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251981511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:41,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:41,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251981511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:41,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T15:18:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:41,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:41,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T15:18:41,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:41,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36156 deadline: 1731251981653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:41,672 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in ba35a6e7578835b556d2567ed9c44ec8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:18:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:41,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251981711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251981711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:41,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251981731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:41,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251981731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:41,788 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/fb9a8130b54340eb8404d6fb63f93745 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fb9a8130b54340eb8404d6fb63f93745 2024-11-10T15:18:41,792 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/A of ba35a6e7578835b556d2567ed9c44ec8 into fb9a8130b54340eb8404d6fb63f93745(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:41,792 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:41,792 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/A, priority=13, startTime=1731251921352; duration=0sec 2024-11-10T15:18:41,792 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:41,792 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:A 2024-11-10T15:18:41,792 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:18:41,793 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:18:41,793 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): ba35a6e7578835b556d2567ed9c44ec8/C is initiating minor compaction (all files) 2024-11-10T15:18:41,793 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba35a6e7578835b556d2567ed9c44ec8/C in TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:41,793 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/2abf30c9f25c46f9a6045ecba6b1af65, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0f6a5116dd2f4f69949d86654542a570, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/18e8c02b336144b7aa99449b0b8bbb58] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp, totalSize=37.1 K 2024-11-10T15:18:41,793 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2abf30c9f25c46f9a6045ecba6b1af65, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731251919320 2024-11-10T15:18:41,793 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f6a5116dd2f4f69949d86654542a570, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731251919429 2024-11-10T15:18:41,794 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18e8c02b336144b7aa99449b0b8bbb58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1731251920635 2024-11-10T15:18:41,800 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba35a6e7578835b556d2567ed9c44ec8#C#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:41,800 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/dce3e6f7de654bac8e3181d27a28b551 is 50, key is test_row_0/C:col10/1731251921303/Put/seqid=0 2024-11-10T15:18:41,803 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/2f82ed07e7a148a684809a1e7f8f0610 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/2f82ed07e7a148a684809a1e7f8f0610 2024-11-10T15:18:41,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T15:18:41,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:41,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:41,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:41,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:41,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:41,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742398_1574 (size=13493) 2024-11-10T15:18:41,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/cde6ed3e4eec4fbdbd98998d63e5d8fc 2024-11-10T15:18:41,812 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/B of ba35a6e7578835b556d2567ed9c44ec8 into 2f82ed07e7a148a684809a1e7f8f0610(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:41,812 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:41,812 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/B, priority=13, startTime=1731251921352; duration=0sec 2024-11-10T15:18:41,812 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:41,812 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:B 2024-11-10T15:18:41,815 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/dce3e6f7de654bac8e3181d27a28b551 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/dce3e6f7de654bac8e3181d27a28b551 2024-11-10T15:18:41,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/6906773d36094adb985ca348115075b0 is 50, key is test_row_0/C:col10/1731251921326/Put/seqid=0 2024-11-10T15:18:41,826 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba35a6e7578835b556d2567ed9c44ec8/C of ba35a6e7578835b556d2567ed9c44ec8 into dce3e6f7de654bac8e3181d27a28b551(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:41,826 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:41,826 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., storeName=ba35a6e7578835b556d2567ed9c44ec8/C, priority=13, startTime=1731251921353; duration=0sec 2024-11-10T15:18:41,827 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:41,827 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba35a6e7578835b556d2567ed9c44ec8:C 2024-11-10T15:18:41,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742399_1575 (size=12301) 2024-11-10T15:18:41,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T15:18:41,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:41,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:41,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:41,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:41,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T15:18:41,998 INFO [Thread-2092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2092. 2024-11-10T15:18:41,998 DEBUG [Thread-2092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:41,998 DEBUG [Thread-2092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:41,998 DEBUG [Thread-2092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:41,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:42,005 INFO [Thread-2094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2094. 2024-11-10T15:18:42,005 DEBUG [Thread-2094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:42,005 DEBUG [Thread-2094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:42,005 DEBUG [Thread-2094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:42,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:42,007 INFO [Thread-2088 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2088. 2024-11-10T15:18:42,007 DEBUG [Thread-2088 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:42,007 DEBUG [Thread-2088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:42,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:42,008 DEBUG [Thread-2088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:42,012 INFO [Thread-2090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2090. 2024-11-10T15:18:42,012 DEBUG [Thread-2090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:42,012 DEBUG [Thread-2090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:42,012 DEBUG [Thread-2090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:42,013 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:42,017 INFO [Thread-2096 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2096. 2024-11-10T15:18:42,017 DEBUG [Thread-2096 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:42,017 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:42,017 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:42,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:42,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:42,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36130 deadline: 1731251982031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36168 deadline: 1731251982031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:42,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:42,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36196 deadline: 1731251982041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36186 deadline: 1731251982041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 , the old value is region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., hostname=c0771061be61,46087,1731251771284, seqNum=2 because the exception is null or not the one we care about 2024-11-10T15:18:42,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T15:18:42,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:42,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:42,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:42,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:42,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:42,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/6906773d36094adb985ca348115075b0 2024-11-10T15:18:42,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/909bda7313a04e5d821a5242c0e94087 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/909bda7313a04e5d821a5242c0e94087 2024-11-10T15:18:42,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/909bda7313a04e5d821a5242c0e94087, entries=150, sequenceid=560, filesize=12.0 K 2024-11-10T15:18:42,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/cde6ed3e4eec4fbdbd98998d63e5d8fc as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/cde6ed3e4eec4fbdbd98998d63e5d8fc 2024-11-10T15:18:42,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/cde6ed3e4eec4fbdbd98998d63e5d8fc, entries=150, sequenceid=560, filesize=12.0 K 2024-11-10T15:18:42,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/6906773d36094adb985ca348115075b0 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/6906773d36094adb985ca348115075b0 2024-11-10T15:18:42,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/6906773d36094adb985ca348115075b0, entries=150, sequenceid=560, filesize=12.0 K 2024-11-10T15:18:42,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ba35a6e7578835b556d2567ed9c44ec8 in 884ms, sequenceid=560, compaction requested=false 2024-11-10T15:18:42,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:42,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T15:18:42,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:42,264 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T15:18:42,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:42,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:42,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:42,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:42,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:42,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:42,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/fbaa5b6d8af14123a04a9f4879cb7487 is 50, key is test_row_0/A:col10/1731251921393/Put/seqid=0 2024-11-10T15:18:42,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742400_1576 (size=12301) 2024-11-10T15:18:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T15:18:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:42,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. as already flushing 2024-11-10T15:18:42,543 INFO [Thread-2079 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2079. 2024-11-10T15:18:42,543 INFO [Thread-2077 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2077. 2024-11-10T15:18:42,543 DEBUG [Thread-2079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:42,543 DEBUG [Thread-2077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:42,543 DEBUG [Thread-2079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:42,543 DEBUG [Thread-2077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:42,543 DEBUG [Thread-2079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:42,543 DEBUG [Thread-2077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:42,543 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:42,544 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:42,552 INFO [Thread-2083 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2083. 2024-11-10T15:18:42,552 INFO [Thread-2085 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2085. 2024-11-10T15:18:42,553 DEBUG [Thread-2083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:42,553 DEBUG [Thread-2085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:42,553 DEBUG [Thread-2083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:42,553 DEBUG [Thread-2085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:42,553 DEBUG [Thread-2083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:42,553 DEBUG [Thread-2085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:42,553 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:42,553 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:42,671 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/fbaa5b6d8af14123a04a9f4879cb7487 2024-11-10T15:18:42,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/75fc42f4fab04a53b2afd1a7a6ba356f is 50, key is test_row_0/B:col10/1731251921393/Put/seqid=0 2024-11-10T15:18:42,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742401_1577 (size=12301) 2024-11-10T15:18:43,079 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/75fc42f4fab04a53b2afd1a7a6ba356f 2024-11-10T15:18:43,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/90c4db188dc74a7f84d6b9457715f902 is 50, key is test_row_0/C:col10/1731251921393/Put/seqid=0 2024-11-10T15:18:43,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742402_1578 (size=12301) 2024-11-10T15:18:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T15:18:43,488 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/90c4db188dc74a7f84d6b9457715f902 2024-11-10T15:18:43,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/fbaa5b6d8af14123a04a9f4879cb7487 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fbaa5b6d8af14123a04a9f4879cb7487 2024-11-10T15:18:43,493 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fbaa5b6d8af14123a04a9f4879cb7487, entries=150, sequenceid=576, filesize=12.0 K 2024-11-10T15:18:43,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/75fc42f4fab04a53b2afd1a7a6ba356f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/75fc42f4fab04a53b2afd1a7a6ba356f 2024-11-10T15:18:43,497 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/75fc42f4fab04a53b2afd1a7a6ba356f, entries=150, sequenceid=576, filesize=12.0 K 2024-11-10T15:18:43,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/90c4db188dc74a7f84d6b9457715f902 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/90c4db188dc74a7f84d6b9457715f902 2024-11-10T15:18:43,500 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/90c4db188dc74a7f84d6b9457715f902, entries=150, sequenceid=576, filesize=12.0 K 2024-11-10T15:18:43,500 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for ba35a6e7578835b556d2567ed9c44ec8 in 1236ms, sequenceid=576, compaction requested=true 2024-11-10T15:18:43,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for ba35a6e7578835b556d2567ed9c44ec8: 2024-11-10T15:18:43,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:43,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-10T15:18:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-10T15:18:43,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-10T15:18:43,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1680 sec 2024-11-10T15:18:43,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.1720 sec 2024-11-10T15:18:43,683 INFO [Thread-2081 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2081. 2024-11-10T15:18:43,683 DEBUG [Thread-2081 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:18:43,683 DEBUG [Thread-2081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:43,683 DEBUG [Thread-2081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:43,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T15:18:45,491 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 261 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 967 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2889 rows 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 968 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2904 rows 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 966 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2892 rows 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 964 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2892 rows 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 966 2024-11-10T15:18:45,491 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2898 rows 2024-11-10T15:18:45,492 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T15:18:45,492 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T15:18:45,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:45,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:45,492 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:18:45,492 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T15:18:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T15:18:45,495 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251925495"}]},"ts":"1731251925495"} 2024-11-10T15:18:45,496 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T15:18:45,496 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T15:18:45,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T15:18:45,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba35a6e7578835b556d2567ed9c44ec8, UNASSIGN}] 2024-11-10T15:18:45,498 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba35a6e7578835b556d2567ed9c44ec8, UNASSIGN 2024-11-10T15:18:45,498 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=ba35a6e7578835b556d2567ed9c44ec8, regionState=CLOSING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:18:45,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba35a6e7578835b556d2567ed9c44ec8, UNASSIGN because future has completed 2024-11-10T15:18:45,500 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T15:18:45,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:18:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T15:18:45,651 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:45,651 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T15:18:45,651 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing ba35a6e7578835b556d2567ed9c44ec8, disabling compactions & flushes 2024-11-10T15:18:45,651 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:45,652 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:45,652 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. after waiting 0 ms 2024-11-10T15:18:45,652 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:45,652 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing ba35a6e7578835b556d2567ed9c44ec8 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T15:18:45,652 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=A 2024-11-10T15:18:45,652 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:45,652 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=B 2024-11-10T15:18:45,652 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:45,652 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba35a6e7578835b556d2567ed9c44ec8, store=C 2024-11-10T15:18:45,652 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:45,655 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/89f6fec346244cd189272c3359469a75 is 50, key is test_row_0/A:col10/1731251923681/Put/seqid=0 2024-11-10T15:18:45,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742403_1579 (size=12301) 2024-11-10T15:18:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T15:18:46,058 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/89f6fec346244cd189272c3359469a75 2024-11-10T15:18:46,064 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/4e129fb8b22c42b89bbe9fceb6ec64df is 50, key is test_row_0/B:col10/1731251923681/Put/seqid=0 2024-11-10T15:18:46,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742404_1580 (size=12301) 2024-11-10T15:18:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T15:18:46,468 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/4e129fb8b22c42b89bbe9fceb6ec64df 2024-11-10T15:18:46,473 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/3bf2dfeec7a24048a6b398fd49c4b1e1 is 50, key is test_row_0/C:col10/1731251923681/Put/seqid=0 2024-11-10T15:18:46,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742405_1581 (size=12301) 2024-11-10T15:18:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T15:18:46,877 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/3bf2dfeec7a24048a6b398fd49c4b1e1 2024-11-10T15:18:46,880 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/A/89f6fec346244cd189272c3359469a75 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/89f6fec346244cd189272c3359469a75 2024-11-10T15:18:46,883 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/89f6fec346244cd189272c3359469a75, entries=150, sequenceid=584, filesize=12.0 K 2024-11-10T15:18:46,884 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/B/4e129fb8b22c42b89bbe9fceb6ec64df as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/4e129fb8b22c42b89bbe9fceb6ec64df 2024-11-10T15:18:46,888 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/4e129fb8b22c42b89bbe9fceb6ec64df, entries=150, sequenceid=584, filesize=12.0 K 2024-11-10T15:18:46,888 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/.tmp/C/3bf2dfeec7a24048a6b398fd49c4b1e1 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/3bf2dfeec7a24048a6b398fd49c4b1e1 2024-11-10T15:18:46,891 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/3bf2dfeec7a24048a6b398fd49c4b1e1, entries=150, sequenceid=584, filesize=12.0 K 2024-11-10T15:18:46,892 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ba35a6e7578835b556d2567ed9c44ec8 in 1240ms, sequenceid=584, compaction requested=true 2024-11-10T15:18:46,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/c9a01ecb6a264642b94ee7c91065132a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/ecf676fb6db94cbdb758bdd3f3a3b03c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/0da2442176a54db3802103b2827cb0eb, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/da7ed475cff444edacb40a47ca3db1e4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/419d61242cde4e818533f0d4ee01d0b1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/529775cc588d430397c09ae9b6fd36e8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/b47c90e9daec4d55b59adcf3c15ca11a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/0644c7032d9548ce98985ea91ef1dd00, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/a54f8d83ece44871be0fbbcd5423d4fa, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/a70cba5e83964a45b0876c951aaecaf1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fbbe5a5c814c43fdba4a39d08d2fd3c5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/714d8730eb9a4050b3e1dcfdab58fb3a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/2b5a3b433e864b979ba1dc0dd0f4ee0e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/03e96dc1babe4368a83f72aca156bad5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/208f154cfed740fe95b57b6831f05236, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/cb1098cdbdd44e38a71509606dc03e14, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/962088b6facb44cc93d36a56a1ae2cfe, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/249c4cf14c70486ca953ee0383f5e7bd, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/207aa53a7cc744bfacfb5c0421cb4960, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/7b0ed8dcc4274318a58980f99978df9e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/f25eb1ba1a074d2dbbc0a34feb1c86d1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/cb1892fac9cf44a8b395cf2248eb6dfa, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/5939707e6f7143108693fe4b48a2d1bf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/53b8358e3e3a44b59fff76f674e9a947, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/ff19aced93ab45bfb47ca8956f6fa182, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/93d73614ae5b4daf8c1a3d7c9d12b942, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/5570834c8e65474ca6e14a6af5323282, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/67853175184b416c884cf927c8b4b86a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fb1276fcf71f44a787068dab86b460d1, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/692a5d7b96f140ffbfd2d74936a453fc, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/467ffc41c9fa4a6484660d73b4f4ed97, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/6656d7882c094064b9c0ae86318a7e6b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/7cda6e0d05fe45be85914ae3f9976ffb, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/bfa6c298a8e546089943dffe1d21ae19, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/937aaea9d0114b84b2f517821ba7c135] to archive 2024-11-10T15:18:46,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:18:46,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/c9a01ecb6a264642b94ee7c91065132a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/c9a01ecb6a264642b94ee7c91065132a 2024-11-10T15:18:46,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/ecf676fb6db94cbdb758bdd3f3a3b03c to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/ecf676fb6db94cbdb758bdd3f3a3b03c 2024-11-10T15:18:46,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/0da2442176a54db3802103b2827cb0eb to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/0da2442176a54db3802103b2827cb0eb 2024-11-10T15:18:46,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/da7ed475cff444edacb40a47ca3db1e4 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/da7ed475cff444edacb40a47ca3db1e4 2024-11-10T15:18:46,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/419d61242cde4e818533f0d4ee01d0b1 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/419d61242cde4e818533f0d4ee01d0b1 2024-11-10T15:18:46,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/529775cc588d430397c09ae9b6fd36e8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/529775cc588d430397c09ae9b6fd36e8 2024-11-10T15:18:46,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/b47c90e9daec4d55b59adcf3c15ca11a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/b47c90e9daec4d55b59adcf3c15ca11a 2024-11-10T15:18:46,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/0644c7032d9548ce98985ea91ef1dd00 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/0644c7032d9548ce98985ea91ef1dd00 2024-11-10T15:18:46,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/a54f8d83ece44871be0fbbcd5423d4fa to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/a54f8d83ece44871be0fbbcd5423d4fa 2024-11-10T15:18:46,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/a70cba5e83964a45b0876c951aaecaf1 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/a70cba5e83964a45b0876c951aaecaf1 2024-11-10T15:18:46,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fbbe5a5c814c43fdba4a39d08d2fd3c5 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fbbe5a5c814c43fdba4a39d08d2fd3c5 2024-11-10T15:18:46,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/714d8730eb9a4050b3e1dcfdab58fb3a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/714d8730eb9a4050b3e1dcfdab58fb3a 2024-11-10T15:18:46,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/2b5a3b433e864b979ba1dc0dd0f4ee0e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/2b5a3b433e864b979ba1dc0dd0f4ee0e 2024-11-10T15:18:46,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/03e96dc1babe4368a83f72aca156bad5 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/03e96dc1babe4368a83f72aca156bad5 2024-11-10T15:18:46,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/208f154cfed740fe95b57b6831f05236 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/208f154cfed740fe95b57b6831f05236 2024-11-10T15:18:46,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/cb1098cdbdd44e38a71509606dc03e14 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/cb1098cdbdd44e38a71509606dc03e14 2024-11-10T15:18:46,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/962088b6facb44cc93d36a56a1ae2cfe to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/962088b6facb44cc93d36a56a1ae2cfe 2024-11-10T15:18:46,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/249c4cf14c70486ca953ee0383f5e7bd to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/249c4cf14c70486ca953ee0383f5e7bd 2024-11-10T15:18:46,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/207aa53a7cc744bfacfb5c0421cb4960 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/207aa53a7cc744bfacfb5c0421cb4960 2024-11-10T15:18:46,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/7b0ed8dcc4274318a58980f99978df9e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/7b0ed8dcc4274318a58980f99978df9e 2024-11-10T15:18:46,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/f25eb1ba1a074d2dbbc0a34feb1c86d1 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/f25eb1ba1a074d2dbbc0a34feb1c86d1 2024-11-10T15:18:46,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/cb1892fac9cf44a8b395cf2248eb6dfa to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/cb1892fac9cf44a8b395cf2248eb6dfa 2024-11-10T15:18:46,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/5939707e6f7143108693fe4b48a2d1bf to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/5939707e6f7143108693fe4b48a2d1bf 2024-11-10T15:18:46,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/53b8358e3e3a44b59fff76f674e9a947 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/53b8358e3e3a44b59fff76f674e9a947 2024-11-10T15:18:46,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/ff19aced93ab45bfb47ca8956f6fa182 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/ff19aced93ab45bfb47ca8956f6fa182 2024-11-10T15:18:46,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/93d73614ae5b4daf8c1a3d7c9d12b942 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/93d73614ae5b4daf8c1a3d7c9d12b942 2024-11-10T15:18:46,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/5570834c8e65474ca6e14a6af5323282 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/5570834c8e65474ca6e14a6af5323282 2024-11-10T15:18:46,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/67853175184b416c884cf927c8b4b86a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/67853175184b416c884cf927c8b4b86a 2024-11-10T15:18:46,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fb1276fcf71f44a787068dab86b460d1 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fb1276fcf71f44a787068dab86b460d1 2024-11-10T15:18:46,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/692a5d7b96f140ffbfd2d74936a453fc to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/692a5d7b96f140ffbfd2d74936a453fc 2024-11-10T15:18:46,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/467ffc41c9fa4a6484660d73b4f4ed97 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/467ffc41c9fa4a6484660d73b4f4ed97 2024-11-10T15:18:46,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/6656d7882c094064b9c0ae86318a7e6b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/6656d7882c094064b9c0ae86318a7e6b 2024-11-10T15:18:46,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/7cda6e0d05fe45be85914ae3f9976ffb to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/7cda6e0d05fe45be85914ae3f9976ffb 2024-11-10T15:18:46,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/bfa6c298a8e546089943dffe1d21ae19 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/bfa6c298a8e546089943dffe1d21ae19 2024-11-10T15:18:46,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/937aaea9d0114b84b2f517821ba7c135 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/937aaea9d0114b84b2f517821ba7c135 2024-11-10T15:18:46,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/9bdb1f75328844df9fd4cf3a125c35c3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/683bd55a977c40f69dbc8470884e59bf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/de0a15de73174c7491ffdb0e34acb337, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/aa3aef0518704095abbf5564f5b3f0ef, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/56e65dd4e8884de39820425c28f9c144, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d81a871d6473490fbd5b8f90209c1080, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/247197642bc64224b3384f3cd4045f87, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/dc5d33b203b14ca4a5edc7a5b41bd172, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/5c6dd6d279c24a359f581e390a0d683a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/f96f746bc7744407993fafc65bc62440, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/b58cd8f3d568449e97fadc0df53cd5e8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/0baade8f83bc42e3adfe86b30a4cfb4c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/5487ad961e6e489a91079c04436ed962, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/29385a7dc5fd43f1a362015c1bd02316, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/ef695c7501624ecf859e5c7e0ab5f4bf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/dc8efd1fe76c4c5296d66eec977a0842, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/ef26de9bd62b4390ac6c89417004cc00, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d95b3128d5214e61b00d4db96cccce71, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/7a5324008f144c8aac3f48b7cdad8926, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/3fb8fe8ff99148569f835c4905d658c5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/2062eb725a7341a699425002a2e60292, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/32a70a457f124046b19fe5a48ad5f0e6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/14d13e816ffa4c04941371bae8b98517, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/392da4a362084ee3a8356c212dbe8ab9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/24bcbfa90d3b4773b38deef87a8fb291, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/03ed8563dc8a4734bd7687300eae603d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/387da5d65bd74cf3a630ff2019fadc58, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/420fb05a7c254fb5b5c1e151d31196bc, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/f9a03fbf3a3c4aaf90d845e0a65f1f2f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/4a7d9862984740889bf42e516cb141d7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/3bf5403086b84bf397df177d4352d042, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/f05a5e013ca84911bbcf2a2758df93de, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/be35670bf5ee412baff2356eda5fdbe2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/eb163e3147174d36932232bddf28e390, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d8248d1b2bc74b398b7e0ad66071709e] to archive 2024-11-10T15:18:46,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:18:46,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/9bdb1f75328844df9fd4cf3a125c35c3 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/9bdb1f75328844df9fd4cf3a125c35c3 2024-11-10T15:18:46,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/683bd55a977c40f69dbc8470884e59bf to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/683bd55a977c40f69dbc8470884e59bf 2024-11-10T15:18:46,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/de0a15de73174c7491ffdb0e34acb337 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/de0a15de73174c7491ffdb0e34acb337 2024-11-10T15:18:46,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/aa3aef0518704095abbf5564f5b3f0ef to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/aa3aef0518704095abbf5564f5b3f0ef 2024-11-10T15:18:46,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/56e65dd4e8884de39820425c28f9c144 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/56e65dd4e8884de39820425c28f9c144 2024-11-10T15:18:46,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d81a871d6473490fbd5b8f90209c1080 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d81a871d6473490fbd5b8f90209c1080 2024-11-10T15:18:46,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/247197642bc64224b3384f3cd4045f87 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/247197642bc64224b3384f3cd4045f87 2024-11-10T15:18:46,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/dc5d33b203b14ca4a5edc7a5b41bd172 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/dc5d33b203b14ca4a5edc7a5b41bd172 2024-11-10T15:18:46,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/5c6dd6d279c24a359f581e390a0d683a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/5c6dd6d279c24a359f581e390a0d683a 2024-11-10T15:18:46,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/f96f746bc7744407993fafc65bc62440 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/f96f746bc7744407993fafc65bc62440 2024-11-10T15:18:46,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/b58cd8f3d568449e97fadc0df53cd5e8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/b58cd8f3d568449e97fadc0df53cd5e8 2024-11-10T15:18:46,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/0baade8f83bc42e3adfe86b30a4cfb4c to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/0baade8f83bc42e3adfe86b30a4cfb4c 2024-11-10T15:18:46,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/5487ad961e6e489a91079c04436ed962 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/5487ad961e6e489a91079c04436ed962 2024-11-10T15:18:46,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/29385a7dc5fd43f1a362015c1bd02316 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/29385a7dc5fd43f1a362015c1bd02316 2024-11-10T15:18:46,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/ef695c7501624ecf859e5c7e0ab5f4bf to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/ef695c7501624ecf859e5c7e0ab5f4bf 2024-11-10T15:18:46,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/dc8efd1fe76c4c5296d66eec977a0842 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/dc8efd1fe76c4c5296d66eec977a0842 2024-11-10T15:18:46,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/ef26de9bd62b4390ac6c89417004cc00 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/ef26de9bd62b4390ac6c89417004cc00 2024-11-10T15:18:46,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d95b3128d5214e61b00d4db96cccce71 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d95b3128d5214e61b00d4db96cccce71 2024-11-10T15:18:46,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/7a5324008f144c8aac3f48b7cdad8926 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/7a5324008f144c8aac3f48b7cdad8926 2024-11-10T15:18:46,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/3fb8fe8ff99148569f835c4905d658c5 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/3fb8fe8ff99148569f835c4905d658c5 2024-11-10T15:18:46,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/2062eb725a7341a699425002a2e60292 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/2062eb725a7341a699425002a2e60292 2024-11-10T15:18:46,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/32a70a457f124046b19fe5a48ad5f0e6 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/32a70a457f124046b19fe5a48ad5f0e6 2024-11-10T15:18:46,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/14d13e816ffa4c04941371bae8b98517 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/14d13e816ffa4c04941371bae8b98517 2024-11-10T15:18:46,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/392da4a362084ee3a8356c212dbe8ab9 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/392da4a362084ee3a8356c212dbe8ab9 2024-11-10T15:18:46,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/24bcbfa90d3b4773b38deef87a8fb291 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/24bcbfa90d3b4773b38deef87a8fb291 2024-11-10T15:18:46,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/03ed8563dc8a4734bd7687300eae603d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/03ed8563dc8a4734bd7687300eae603d 2024-11-10T15:18:46,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/387da5d65bd74cf3a630ff2019fadc58 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/387da5d65bd74cf3a630ff2019fadc58 2024-11-10T15:18:46,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/420fb05a7c254fb5b5c1e151d31196bc to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/420fb05a7c254fb5b5c1e151d31196bc 2024-11-10T15:18:46,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/f9a03fbf3a3c4aaf90d845e0a65f1f2f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/f9a03fbf3a3c4aaf90d845e0a65f1f2f 2024-11-10T15:18:46,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/4a7d9862984740889bf42e516cb141d7 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/4a7d9862984740889bf42e516cb141d7 2024-11-10T15:18:46,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/3bf5403086b84bf397df177d4352d042 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/3bf5403086b84bf397df177d4352d042 2024-11-10T15:18:46,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/f05a5e013ca84911bbcf2a2758df93de to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/f05a5e013ca84911bbcf2a2758df93de 2024-11-10T15:18:46,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/be35670bf5ee412baff2356eda5fdbe2 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/be35670bf5ee412baff2356eda5fdbe2 2024-11-10T15:18:46,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/eb163e3147174d36932232bddf28e390 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/eb163e3147174d36932232bddf28e390 2024-11-10T15:18:46,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d8248d1b2bc74b398b7e0ad66071709e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/d8248d1b2bc74b398b7e0ad66071709e 2024-11-10T15:18:46,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/a587d086b09c4919a3eb718be7dae653, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/b01050292be040849bc1bf413cbdbeae, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/5608ca45d8894f6d8a30f1d367ab7301, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/54fff8a76e614663ac91eec637cf2183, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/125830b1fbed4e2895ac5aea96702a02, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0b933cc3cea249b791bb8f1b5b9633d0, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/f62a066ea8544ba688196205e3f30c3f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/cabe29218f364b59a96a955328796578, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d5893eca73d0448d833409464b241a3a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/e9f9d3e39cdb40ebbce65f29188b9011, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/8a7b168e522c4145b30b62e246fd948d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/78362d9237df4d85a882167ca408eed7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/14cbad89502841bcb1d1dabe437f5273, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/24d1f255d46a4e768143b177f450362e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c799de7f03294277b924553dd30f23bc, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/8ba7d739e97c46f7834a3fb2f43cb5e7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/550e63b1d672483fbcff3b5e6ce0aeb2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d0a1af65af0146329f30b64ee4d4f3fb, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c1f1d623a68d4410901ff4e6e414fd84, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c917c2d756ab476fab9a217db998349c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/3147038e25cd413f85e9dffaa6fe1e56, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/bad08b648f5b42999360d8e430ca9bd9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/72bf646dfd97464e8ff98124fec306b9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/fd9b7aed755649dda828f143799795cf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/13e9816f59ce47d4aafa542a1dff10f2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0ed78e74a06c4336ab84cd57c569bc38, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/2eef1211179e4d799e0b87686a1caf56, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d759cd192e3a44e9a2de9a1d0b51b5d5, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/ae499559a9bb491d8ca12868309c4141, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/f4557191085343368b455835bf4e7479, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/299a9444ed2847a6bb7fa30cdc820142, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/2abf30c9f25c46f9a6045ecba6b1af65, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/045b671cbce34734a4ec90d86ac28548, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0f6a5116dd2f4f69949d86654542a570, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/18e8c02b336144b7aa99449b0b8bbb58] to archive 2024-11-10T15:18:46,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:18:46,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/a587d086b09c4919a3eb718be7dae653 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/a587d086b09c4919a3eb718be7dae653 2024-11-10T15:18:46,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/b01050292be040849bc1bf413cbdbeae to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/b01050292be040849bc1bf413cbdbeae 2024-11-10T15:18:46,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/5608ca45d8894f6d8a30f1d367ab7301 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/5608ca45d8894f6d8a30f1d367ab7301 2024-11-10T15:18:46,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/54fff8a76e614663ac91eec637cf2183 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/54fff8a76e614663ac91eec637cf2183 2024-11-10T15:18:46,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/125830b1fbed4e2895ac5aea96702a02 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/125830b1fbed4e2895ac5aea96702a02 2024-11-10T15:18:46,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0b933cc3cea249b791bb8f1b5b9633d0 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0b933cc3cea249b791bb8f1b5b9633d0 2024-11-10T15:18:46,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/f62a066ea8544ba688196205e3f30c3f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/f62a066ea8544ba688196205e3f30c3f 2024-11-10T15:18:46,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/cabe29218f364b59a96a955328796578 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/cabe29218f364b59a96a955328796578 2024-11-10T15:18:46,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d5893eca73d0448d833409464b241a3a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d5893eca73d0448d833409464b241a3a 2024-11-10T15:18:46,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/e9f9d3e39cdb40ebbce65f29188b9011 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/e9f9d3e39cdb40ebbce65f29188b9011 2024-11-10T15:18:46,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/8a7b168e522c4145b30b62e246fd948d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/8a7b168e522c4145b30b62e246fd948d 2024-11-10T15:18:46,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/78362d9237df4d85a882167ca408eed7 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/78362d9237df4d85a882167ca408eed7 2024-11-10T15:18:46,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/14cbad89502841bcb1d1dabe437f5273 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/14cbad89502841bcb1d1dabe437f5273 2024-11-10T15:18:46,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/24d1f255d46a4e768143b177f450362e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/24d1f255d46a4e768143b177f450362e 2024-11-10T15:18:46,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c799de7f03294277b924553dd30f23bc to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c799de7f03294277b924553dd30f23bc 2024-11-10T15:18:46,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/8ba7d739e97c46f7834a3fb2f43cb5e7 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/8ba7d739e97c46f7834a3fb2f43cb5e7 2024-11-10T15:18:46,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/550e63b1d672483fbcff3b5e6ce0aeb2 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/550e63b1d672483fbcff3b5e6ce0aeb2 2024-11-10T15:18:46,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d0a1af65af0146329f30b64ee4d4f3fb to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d0a1af65af0146329f30b64ee4d4f3fb 2024-11-10T15:18:46,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c1f1d623a68d4410901ff4e6e414fd84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c1f1d623a68d4410901ff4e6e414fd84 2024-11-10T15:18:46,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c917c2d756ab476fab9a217db998349c to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/c917c2d756ab476fab9a217db998349c 2024-11-10T15:18:46,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/3147038e25cd413f85e9dffaa6fe1e56 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/3147038e25cd413f85e9dffaa6fe1e56 2024-11-10T15:18:46,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/bad08b648f5b42999360d8e430ca9bd9 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/bad08b648f5b42999360d8e430ca9bd9 2024-11-10T15:18:46,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/72bf646dfd97464e8ff98124fec306b9 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/72bf646dfd97464e8ff98124fec306b9 2024-11-10T15:18:46,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/fd9b7aed755649dda828f143799795cf to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/fd9b7aed755649dda828f143799795cf 2024-11-10T15:18:46,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/13e9816f59ce47d4aafa542a1dff10f2 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/13e9816f59ce47d4aafa542a1dff10f2 2024-11-10T15:18:46,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0ed78e74a06c4336ab84cd57c569bc38 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0ed78e74a06c4336ab84cd57c569bc38 2024-11-10T15:18:46,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/2eef1211179e4d799e0b87686a1caf56 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/2eef1211179e4d799e0b87686a1caf56 2024-11-10T15:18:46,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d759cd192e3a44e9a2de9a1d0b51b5d5 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/d759cd192e3a44e9a2de9a1d0b51b5d5 2024-11-10T15:18:46,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/ae499559a9bb491d8ca12868309c4141 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/ae499559a9bb491d8ca12868309c4141 2024-11-10T15:18:46,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/f4557191085343368b455835bf4e7479 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/f4557191085343368b455835bf4e7479 2024-11-10T15:18:46,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/299a9444ed2847a6bb7fa30cdc820142 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/299a9444ed2847a6bb7fa30cdc820142 2024-11-10T15:18:46,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/2abf30c9f25c46f9a6045ecba6b1af65 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/2abf30c9f25c46f9a6045ecba6b1af65 2024-11-10T15:18:46,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/045b671cbce34734a4ec90d86ac28548 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/045b671cbce34734a4ec90d86ac28548 2024-11-10T15:18:46,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0f6a5116dd2f4f69949d86654542a570 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/0f6a5116dd2f4f69949d86654542a570 2024-11-10T15:18:46,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/18e8c02b336144b7aa99449b0b8bbb58 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/18e8c02b336144b7aa99449b0b8bbb58 2024-11-10T15:18:46,993 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/recovered.edits/587.seqid, newMaxSeqId=587, maxSeqId=1 2024-11-10T15:18:46,993 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. 2024-11-10T15:18:46,994 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for ba35a6e7578835b556d2567ed9c44ec8: Waiting for close lock at 1731251925651Running coprocessor pre-close hooks at 1731251925651Disabling compacts and flushes for region at 1731251925651Disabling writes for close at 1731251925652 (+1 ms)Obtaining lock to block concurrent updates at 1731251925652Preparing flush snapshotting stores in ba35a6e7578835b556d2567ed9c44ec8 at 1731251925652Finished memstore snapshotting TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731251925652Flushing stores of TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8. at 1731251925652Flushing ba35a6e7578835b556d2567ed9c44ec8/A: creating writer at 1731251925652Flushing ba35a6e7578835b556d2567ed9c44ec8/A: appending metadata at 1731251925654 (+2 ms)Flushing ba35a6e7578835b556d2567ed9c44ec8/A: closing flushed file at 1731251925654Flushing ba35a6e7578835b556d2567ed9c44ec8/B: creating writer at 1731251926061 (+407 ms)Flushing ba35a6e7578835b556d2567ed9c44ec8/B: appending metadata at 1731251926063 (+2 ms)Flushing ba35a6e7578835b556d2567ed9c44ec8/B: closing flushed file at 1731251926063Flushing ba35a6e7578835b556d2567ed9c44ec8/C: creating writer at 1731251926471 (+408 ms)Flushing ba35a6e7578835b556d2567ed9c44ec8/C: appending metadata at 1731251926473 (+2 ms)Flushing ba35a6e7578835b556d2567ed9c44ec8/C: closing flushed file at 1731251926473Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@724bbdcd: reopening flushed file at 1731251926879 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c451fb1: reopening flushed file at 1731251926883 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40dc1199: reopening flushed file at 1731251926888 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ba35a6e7578835b556d2567ed9c44ec8 in 1240ms, sequenceid=584, compaction requested=true at 1731251926892 (+4 ms)Writing region close event to WAL at 1731251926990 (+98 ms)Running coprocessor post-close hooks at 1731251926993 (+3 ms)Closed at 1731251926993 2024-11-10T15:18:46,996 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:46,996 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=ba35a6e7578835b556d2567ed9c44ec8, regionState=CLOSED 2024-11-10T15:18:46,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:18:47,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-10T15:18:47,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure ba35a6e7578835b556d2567ed9c44ec8, server=c0771061be61,46087,1731251771284 in 1.4980 sec 2024-11-10T15:18:47,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-10T15:18:47,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba35a6e7578835b556d2567ed9c44ec8, UNASSIGN in 1.5030 sec 2024-11-10T15:18:47,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-10T15:18:47,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5060 sec 2024-11-10T15:18:47,005 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251927005"}]},"ts":"1731251927005"} 2024-11-10T15:18:47,006 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T15:18:47,006 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T15:18:47,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5140 sec 2024-11-10T15:18:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T15:18:47,641 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:47,642 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T15:18:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:47,643 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-10T15:18:47,644 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:47,645 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:47,647 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/recovered.edits] 2024-11-10T15:18:47,650 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/89f6fec346244cd189272c3359469a75 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/89f6fec346244cd189272c3359469a75 2024-11-10T15:18:47,651 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/909bda7313a04e5d821a5242c0e94087 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/909bda7313a04e5d821a5242c0e94087 2024-11-10T15:18:47,652 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fb9a8130b54340eb8404d6fb63f93745 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fb9a8130b54340eb8404d6fb63f93745 2024-11-10T15:18:47,653 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fbaa5b6d8af14123a04a9f4879cb7487 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/A/fbaa5b6d8af14123a04a9f4879cb7487 2024-11-10T15:18:47,655 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/2f82ed07e7a148a684809a1e7f8f0610 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/2f82ed07e7a148a684809a1e7f8f0610 2024-11-10T15:18:47,656 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/4e129fb8b22c42b89bbe9fceb6ec64df to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/4e129fb8b22c42b89bbe9fceb6ec64df 2024-11-10T15:18:47,657 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/75fc42f4fab04a53b2afd1a7a6ba356f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/75fc42f4fab04a53b2afd1a7a6ba356f 2024-11-10T15:18:47,659 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/cde6ed3e4eec4fbdbd98998d63e5d8fc to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/B/cde6ed3e4eec4fbdbd98998d63e5d8fc 2024-11-10T15:18:47,661 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/3bf2dfeec7a24048a6b398fd49c4b1e1 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/3bf2dfeec7a24048a6b398fd49c4b1e1 2024-11-10T15:18:47,662 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/6906773d36094adb985ca348115075b0 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/6906773d36094adb985ca348115075b0 2024-11-10T15:18:47,662 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/90c4db188dc74a7f84d6b9457715f902 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/90c4db188dc74a7f84d6b9457715f902 2024-11-10T15:18:47,663 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/dce3e6f7de654bac8e3181d27a28b551 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/C/dce3e6f7de654bac8e3181d27a28b551 2024-11-10T15:18:47,666 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/recovered.edits/587.seqid to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8/recovered.edits/587.seqid 2024-11-10T15:18:47,666 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/ba35a6e7578835b556d2567ed9c44ec8 2024-11-10T15:18:47,667 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T15:18:47,669 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:47,671 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T15:18:47,672 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T15:18:47,674 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:47,674 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T15:18:47,674 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731251927674"}]},"ts":"9223372036854775807"} 2024-11-10T15:18:47,676 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T15:18:47,676 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ba35a6e7578835b556d2567ed9c44ec8, NAME => 'TestAcidGuarantees,,1731251899762.ba35a6e7578835b556d2567ed9c44ec8.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T15:18:47,676 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T15:18:47,676 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731251927676"}]},"ts":"9223372036854775807"} 2024-11-10T15:18:47,678 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T15:18:47,679 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:47,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 37 msec 2024-11-10T15:18:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-10T15:18:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T15:18:47,751 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:47,759 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251 (was 251), OpenFileDescriptor=451 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=823 (was 946), ProcessCount=11 (was 11), AvailableMemoryMB=2507 (was 3640) 2024-11-10T15:18:47,768 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=823, ProcessCount=11, AvailableMemoryMB=2507 2024-11-10T15:18:47,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T15:18:47,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T15:18:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:47,771 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T15:18:47,771 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:47,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-10T15:18:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T15:18:47,772 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T15:18:47,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742406_1582 (size=1061) 2024-11-10T15:18:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T15:18:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T15:18:48,178 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85 2024-11-10T15:18:48,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742407_1583 (size=53) 2024-11-10T15:18:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T15:18:48,583 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:18:48,583 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f49fc5144c71cae6bffc0a2f5b02ce84, disabling compactions & flushes 2024-11-10T15:18:48,583 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:48,583 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:48,583 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. after waiting 0 ms 2024-11-10T15:18:48,583 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:48,583 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:48,583 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f49fc5144c71cae6bffc0a2f5b02ce84: Waiting for close lock at 1731251928583Disabling compacts and flushes for region at 1731251928583Disabling writes for close at 1731251928583Writing region close event to WAL at 1731251928583Closed at 1731251928583 2024-11-10T15:18:48,584 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T15:18:48,585 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731251928585"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731251928585"}]},"ts":"1731251928585"} 2024-11-10T15:18:48,586 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T15:18:48,587 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T15:18:48,587 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251928587"}]},"ts":"1731251928587"} 2024-11-10T15:18:48,588 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T15:18:48,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f49fc5144c71cae6bffc0a2f5b02ce84, ASSIGN}] 2024-11-10T15:18:48,589 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f49fc5144c71cae6bffc0a2f5b02ce84, ASSIGN 2024-11-10T15:18:48,589 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f49fc5144c71cae6bffc0a2f5b02ce84, ASSIGN; state=OFFLINE, location=c0771061be61,46087,1731251771284; forceNewPlan=false, retain=false 2024-11-10T15:18:48,740 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=f49fc5144c71cae6bffc0a2f5b02ce84, regionState=OPENING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:18:48,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f49fc5144c71cae6bffc0a2f5b02ce84, ASSIGN because future has completed 2024-11-10T15:18:48,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:18:48,898 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:48,898 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} 2024-11-10T15:18:48,898 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:48,898 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:18:48,899 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:48,899 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:48,900 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:48,900 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:18:48,901 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f49fc5144c71cae6bffc0a2f5b02ce84 columnFamilyName A 2024-11-10T15:18:48,901 DEBUG [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:48,901 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.HStore(327): Store=f49fc5144c71cae6bffc0a2f5b02ce84/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:18:48,901 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:48,902 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:18:48,902 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f49fc5144c71cae6bffc0a2f5b02ce84 columnFamilyName B 2024-11-10T15:18:48,902 DEBUG [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:48,903 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.HStore(327): Store=f49fc5144c71cae6bffc0a2f5b02ce84/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:18:48,903 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:48,903 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:18:48,903 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f49fc5144c71cae6bffc0a2f5b02ce84 columnFamilyName C 2024-11-10T15:18:48,904 DEBUG [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:48,904 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.HStore(327): Store=f49fc5144c71cae6bffc0a2f5b02ce84/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:18:48,904 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:48,904 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:48,904 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:48,905 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:48,905 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:48,905 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:48,906 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T15:18:48,907 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:48,908 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T15:18:48,909 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened f49fc5144c71cae6bffc0a2f5b02ce84; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62918841, jitterRate=-0.06243620812892914}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T15:18:48,909 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:48,909 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for f49fc5144c71cae6bffc0a2f5b02ce84: Running coprocessor pre-open hook at 1731251928899Writing region info on filesystem at 1731251928899Initializing all the Stores at 1731251928899Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251928899Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251928899Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251928900 (+1 ms)Cleaning up temporary data from old regions at 1731251928905 (+5 ms)Running coprocessor post-open hooks at 1731251928909 (+4 ms)Region opened successfully at 1731251928909 2024-11-10T15:18:48,910 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., pid=148, masterSystemTime=1731251928895 2024-11-10T15:18:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T15:18:48,912 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:48,912 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:48,913 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=f49fc5144c71cae6bffc0a2f5b02ce84, regionState=OPEN, openSeqNum=2, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:18:48,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:18:48,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-10T15:18:48,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 in 171 msec 2024-11-10T15:18:48,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-10T15:18:48,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f49fc5144c71cae6bffc0a2f5b02ce84, ASSIGN in 329 msec 2024-11-10T15:18:48,919 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T15:18:48,919 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251928919"}]},"ts":"1731251928919"} 2024-11-10T15:18:48,920 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T15:18:48,921 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T15:18:48,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-10T15:18:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T15:18:49,922 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:49,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e76e6cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:49,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:49,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:49,924 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2307099d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:49,925 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@965511b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:49,926 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:49,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:49,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:49,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:49,927 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T15:18:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is c0771061be61,41501,1731251770419 2024-11-10T15:18:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2bea402c 2024-11-10T15:18:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T15:18:49,929 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T15:18:49,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T15:18:49,930 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T15:18:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T15:18:49,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742408_1584 (size=1097) 2024-11-10T15:18:50,341 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T15:18:50,341 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T15:18:50,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T15:18:50,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f49fc5144c71cae6bffc0a2f5b02ce84, REOPEN/MOVE}] 2024-11-10T15:18:50,346 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f49fc5144c71cae6bffc0a2f5b02ce84, REOPEN/MOVE 2024-11-10T15:18:50,347 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=f49fc5144c71cae6bffc0a2f5b02ce84, regionState=CLOSING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:18:50,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f49fc5144c71cae6bffc0a2f5b02ce84, REOPEN/MOVE because future has completed 2024-11-10T15:18:50,348 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T15:18:50,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:18:50,500 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:50,500 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T15:18:50,500 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing f49fc5144c71cae6bffc0a2f5b02ce84, disabling compactions & flushes 2024-11-10T15:18:50,500 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:50,500 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:50,500 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. after waiting 0 ms 2024-11-10T15:18:50,500 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:50,503 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T15:18:50,503 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:50,503 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for f49fc5144c71cae6bffc0a2f5b02ce84: Waiting for close lock at 1731251930500Running coprocessor pre-close hooks at 1731251930500Disabling compacts and flushes for region at 1731251930500Disabling writes for close at 1731251930500Writing region close event to WAL at 1731251930500Running coprocessor post-close hooks at 1731251930503 (+3 ms)Closed at 1731251930503 2024-11-10T15:18:50,503 WARN [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: f49fc5144c71cae6bffc0a2f5b02ce84 to self. 2024-11-10T15:18:50,505 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:50,505 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=f49fc5144c71cae6bffc0a2f5b02ce84, regionState=CLOSED 2024-11-10T15:18:50,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:18:50,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-10T15:18:50,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 in 159 msec 2024-11-10T15:18:50,509 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f49fc5144c71cae6bffc0a2f5b02ce84, REOPEN/MOVE; state=CLOSED, location=c0771061be61,46087,1731251771284; forceNewPlan=false, retain=true 2024-11-10T15:18:50,660 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=f49fc5144c71cae6bffc0a2f5b02ce84, regionState=OPENING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:18:50,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f49fc5144c71cae6bffc0a2f5b02ce84, REOPEN/MOVE because future has completed 2024-11-10T15:18:50,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:18:50,815 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:50,816 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} 2024-11-10T15:18:50,816 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:50,816 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T15:18:50,816 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:50,816 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:50,817 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:50,817 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:18:50,818 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f49fc5144c71cae6bffc0a2f5b02ce84 columnFamilyName A 2024-11-10T15:18:50,819 DEBUG [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:50,819 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.HStore(327): Store=f49fc5144c71cae6bffc0a2f5b02ce84/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:18:50,819 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:50,820 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:18:50,820 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f49fc5144c71cae6bffc0a2f5b02ce84 columnFamilyName B 2024-11-10T15:18:50,820 DEBUG [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:50,821 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.HStore(327): Store=f49fc5144c71cae6bffc0a2f5b02ce84/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:18:50,821 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:50,821 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T15:18:50,821 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f49fc5144c71cae6bffc0a2f5b02ce84 columnFamilyName C 2024-11-10T15:18:50,821 DEBUG [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:50,822 INFO [StoreOpener-f49fc5144c71cae6bffc0a2f5b02ce84-1 {}] regionserver.HStore(327): Store=f49fc5144c71cae6bffc0a2f5b02ce84/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T15:18:50,822 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:50,822 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:50,822 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:50,823 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:50,824 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:50,824 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:50,824 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T15:18:50,825 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:50,826 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened f49fc5144c71cae6bffc0a2f5b02ce84; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68070445, jitterRate=0.014328673481941223}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T15:18:50,826 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:50,827 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for f49fc5144c71cae6bffc0a2f5b02ce84: Running coprocessor pre-open hook at 1731251930816Writing region info on filesystem at 1731251930816Initializing all the Stores at 1731251930817 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251930817Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251930817Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731251930817Cleaning up temporary data from old regions at 1731251930824 (+7 ms)Running coprocessor post-open hooks at 1731251930826 (+2 ms)Region opened successfully at 1731251930827 (+1 ms) 2024-11-10T15:18:50,827 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., pid=153, masterSystemTime=1731251930813 2024-11-10T15:18:50,829 DEBUG [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:50,829 INFO [RS_OPEN_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:50,829 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=f49fc5144c71cae6bffc0a2f5b02ce84, regionState=OPEN, openSeqNum=5, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:18:50,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:18:50,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-10T15:18:50,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 in 170 msec 2024-11-10T15:18:50,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-10T15:18:50,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f49fc5144c71cae6bffc0a2f5b02ce84, REOPEN/MOVE in 486 msec 2024-11-10T15:18:50,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-10T15:18:50,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-11-10T15:18:50,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-10T15:18:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-10T15:18:50,839 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:50,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,839 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:50,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:50,840 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d606323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:50,841 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d44f83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,842 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:50,842 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:50,843 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26556a1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:50,844 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@644469f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:50,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:50,845 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fbf66bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:50,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:50,846 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:50,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ce59665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,847 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:50,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:50,848 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:50,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:50,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:50,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d12cf5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:50,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:50,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:50,849 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77b41d10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,850 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:50,850 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:50,850 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a08d6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:50,851 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180dcd61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,852 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:50,852 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:50,853 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f65d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:50,854 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39af97d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,855 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:50,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:50,856 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:50,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:50,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:50,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74cce6e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:50,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:50,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:50,857 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e168229, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,858 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:50,858 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:50,859 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f0cd557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:50,861 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f39d29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,862 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:50,862 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:50,863 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7251fb53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:50,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:50,864 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:50,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f5222, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c58aaf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,866 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0771061be61,41501,-1 for getting cluster id 2024-11-10T15:18:50,866 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T15:18:50,867 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58ceec01-b1f5-4676-9924-48a824d2685c' 2024-11-10T15:18:50,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T15:18:50,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58ceec01-b1f5-4676-9924-48a824d2685c" 2024-11-10T15:18:50,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1956534, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0771061be61,41501,-1] 2024-11-10T15:18:50,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T15:18:50,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:18:50,868 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T15:18:50,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41aa46da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T15:18:50,879 DEBUG [Thread-2599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:50,879 DEBUG [Thread-2603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:50,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:50,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:50,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:50,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:50,880 DEBUG [Thread-2597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:50,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:50,882 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:18:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:50,883 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:18:50,887 DEBUG [Thread-2588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:50,887 DEBUG [Thread-2592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:50,887 DEBUG [Thread-2594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:50,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:50,888 DEBUG [Thread-2601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:50,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:50,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:50,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:18:50,889 DEBUG [Thread-2586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:50,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:50,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:50,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:50,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-10T15:18:50,890 DEBUG [Thread-2586 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:50,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T15:18:50,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:50,891 DEBUG [Thread-2590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:50,891 DEBUG [Thread-2586 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:50,891 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:50,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:18:50,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:18:50,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:50,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:50,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:18:50,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:50,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:50,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:50,894 DEBUG [Thread-2605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T15:18:50,894 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0771061be61,46087,1731251771284, seqNum=-1] 2024-11-10T15:18:50,895 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:50,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T15:18:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:18:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:18:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:18:50,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T15:18:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5] 2024-11-10T15:18:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:50,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T15:18:50,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:18:50,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:50,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:18:50,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:50,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:18:50,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:50,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100de8636e0db7484f92e44a5fe5bc471c_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251930907/Put/seqid=0 2024-11-10T15:18:50,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742409_1585 (size=12154) 2024-11-10T15:18:50,999 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T15:18:51,006 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100de8636e0db7484f92e44a5fe5bc471c_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100de8636e0db7484f92e44a5fe5bc471c_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:51,007 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/a293088abd0d408881a57626a2f53dbe, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:51,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/a293088abd0d408881a57626a2f53dbe is 175, key is test_row_0/A:col10/1731251930907/Put/seqid=0 2024-11-10T15:18:51,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251930991, max timestamp=1731251931006Num uniques 859; 2024-11-10T15:18:51,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:18:51,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251930991, max timestamp=1731251931005Num uniques 859; 2024-11-10T15:18:51,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:18:51,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251930991, max timestamp=1731251931006Num uniques 859; 2024-11-10T15:18:51,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:18:51,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742410_1586 (size=30955) 2024-11-10T15:18:51,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251991029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251991030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251991035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251991037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251991038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T15:18:51,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:51,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:51,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:51,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251991141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251991141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251991142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251991152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251991152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T15:18:51,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:51,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:51,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:51,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T15:18:51,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T15:18:51,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:51,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:51,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251991351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251991351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251991351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,356 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,356 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,356 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251991361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251991361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,428 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/a293088abd0d408881a57626a2f53dbe 2024-11-10T15:18:51,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/0bd2d43c5ec842aa8234b61116a6acf6 is 50, key is test_row_0/B:col10/1731251930907/Put/seqid=0 2024-11-10T15:18:51,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742411_1587 (size=12001) 2024-11-10T15:18:51,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T15:18:51,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:51,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:51,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:51,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T15:18:51,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T15:18:51,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:51,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:51,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:51,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251991661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251991662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251991662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251991671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:51,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251991671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:51,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:51,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:51,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T15:18:51,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:51,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:51,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:51,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:51,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/0bd2d43c5ec842aa8234b61116a6acf6 2024-11-10T15:18:51,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/c08eb21eb71d4d6d8f579fb7b778d887 is 50, key is test_row_0/C:col10/1731251930907/Put/seqid=0 2024-11-10T15:18:51,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742412_1588 (size=12001) 2024-11-10T15:18:51,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/c08eb21eb71d4d6d8f579fb7b778d887 2024-11-10T15:18:51,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/a293088abd0d408881a57626a2f53dbe as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/a293088abd0d408881a57626a2f53dbe 2024-11-10T15:18:51,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/a293088abd0d408881a57626a2f53dbe, entries=150, sequenceid=15, filesize=30.2 K 2024-11-10T15:18:51,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/0bd2d43c5ec842aa8234b61116a6acf6 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0bd2d43c5ec842aa8234b61116a6acf6 2024-11-10T15:18:51,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0bd2d43c5ec842aa8234b61116a6acf6, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T15:18:51,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/c08eb21eb71d4d6d8f579fb7b778d887 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c08eb21eb71d4d6d8f579fb7b778d887 2024-11-10T15:18:51,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c08eb21eb71d4d6d8f579fb7b778d887, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T15:18:51,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for f49fc5144c71cae6bffc0a2f5b02ce84 in 984ms, sequenceid=15, compaction requested=false 2024-11-10T15:18:51,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:18:51,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T15:18:51,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:51,966 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-10T15:18:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:18:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:18:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:18:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:52,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109e505b60600e4bf09b9603352485becd_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251931036/Put/seqid=0 2024-11-10T15:18:52,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T15:18:52,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742413_1589 (size=12154) 2024-11-10T15:18:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:52,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:52,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251992186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251992186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251992188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251992188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251992189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251992302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251992302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251992302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251992302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251992302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:52,435 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109e505b60600e4bf09b9603352485becd_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109e505b60600e4bf09b9603352485becd_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:52,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/85da13d91f554d62b066109173de538b, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:52,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/85da13d91f554d62b066109173de538b is 175, key is test_row_0/A:col10/1731251931036/Put/seqid=0 2024-11-10T15:18:52,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742414_1590 (size=30955) 2024-11-10T15:18:52,446 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/85da13d91f554d62b066109173de538b 2024-11-10T15:18:52,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/affec116f2cb48fba7e02af1affc1189 is 50, key is test_row_0/B:col10/1731251931036/Put/seqid=0 2024-11-10T15:18:52,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742415_1591 (size=12001) 2024-11-10T15:18:52,468 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/affec116f2cb48fba7e02af1affc1189 2024-11-10T15:18:52,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/ca834afc4b4e4a7984df068a6d2d7fdf is 50, key is test_row_0/C:col10/1731251931036/Put/seqid=0 2024-11-10T15:18:52,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742416_1592 (size=12001) 2024-11-10T15:18:52,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251992512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251992512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251992512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251992512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251992512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,519 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T15:18:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251992821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251992821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251992821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251992822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251992822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:52,882 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/ca834afc4b4e4a7984df068a6d2d7fdf 2024-11-10T15:18:52,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/85da13d91f554d62b066109173de538b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/85da13d91f554d62b066109173de538b 2024-11-10T15:18:52,897 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/85da13d91f554d62b066109173de538b, entries=150, sequenceid=52, filesize=30.2 K 2024-11-10T15:18:52,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/affec116f2cb48fba7e02af1affc1189 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/affec116f2cb48fba7e02af1affc1189 2024-11-10T15:18:52,901 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/affec116f2cb48fba7e02af1affc1189, entries=150, sequenceid=52, filesize=11.7 K 2024-11-10T15:18:52,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/ca834afc4b4e4a7984df068a6d2d7fdf as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ca834afc4b4e4a7984df068a6d2d7fdf 2024-11-10T15:18:52,906 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ca834afc4b4e4a7984df068a6d2d7fdf, entries=150, sequenceid=52, filesize=11.7 K 2024-11-10T15:18:52,907 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for f49fc5144c71cae6bffc0a2f5b02ce84 in 941ms, sequenceid=52, compaction requested=false 2024-11-10T15:18:52,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:18:52,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:52,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-10T15:18:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-10T15:18:52,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-10T15:18:52,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0140 sec 2024-11-10T15:18:52,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.0220 sec 2024-11-10T15:18:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T15:18:53,042 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:53,047 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:53,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-10T15:18:53,048 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:53,049 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:53,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T15:18:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T15:18:53,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T15:18:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:53,203 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T15:18:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:18:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:18:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:18:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:53,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111001300e66afc74296bd65ff483edfe5b8_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251932183/Put/seqid=0 2024-11-10T15:18:53,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742417_1593 (size=12154) 2024-11-10T15:18:53,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:53,224 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111001300e66afc74296bd65ff483edfe5b8_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111001300e66afc74296bd65ff483edfe5b8_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:53,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/b7eb39191ea44698b97d0489a0626a9f, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:53,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/b7eb39191ea44698b97d0489a0626a9f is 175, key is test_row_0/A:col10/1731251932183/Put/seqid=0 2024-11-10T15:18:53,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742418_1594 (size=30955) 2024-11-10T15:18:53,238 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/b7eb39191ea44698b97d0489a0626a9f 2024-11-10T15:18:53,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/0a43b027dc604fb29339e5fc82ffdd51 is 50, key is test_row_0/B:col10/1731251932183/Put/seqid=0 2024-11-10T15:18:53,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742419_1595 (size=12001) 2024-11-10T15:18:53,250 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/0a43b027dc604fb29339e5fc82ffdd51 2024-11-10T15:18:53,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/8dbe2c7a74854f99a86f5d03d06939ea is 50, key is test_row_0/C:col10/1731251932183/Put/seqid=0 2024-11-10T15:18:53,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742420_1596 (size=12001) 2024-11-10T15:18:53,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:53,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251933349, max timestamp=1731251933356Num uniques 859; 2024-11-10T15:18:53,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251933349, max timestamp=1731251933356Num uniques 859; 2024-11-10T15:18:53,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:18:53,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:18:53,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251933349, max timestamp=1731251933356Num uniques 859; 2024-11-10T15:18:53,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:18:53,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T15:18:53,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251993379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251993379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251993380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251993385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251993386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251993491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251993491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251993492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251993502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251993502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T15:18:53,688 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/8dbe2c7a74854f99a86f5d03d06939ea 2024-11-10T15:18:53,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/b7eb39191ea44698b97d0489a0626a9f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/b7eb39191ea44698b97d0489a0626a9f 2024-11-10T15:18:53,694 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/b7eb39191ea44698b97d0489a0626a9f, entries=150, sequenceid=62, filesize=30.2 K 2024-11-10T15:18:53,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/0a43b027dc604fb29339e5fc82ffdd51 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0a43b027dc604fb29339e5fc82ffdd51 2024-11-10T15:18:53,697 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0a43b027dc604fb29339e5fc82ffdd51, entries=150, sequenceid=62, filesize=11.7 K 2024-11-10T15:18:53,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/8dbe2c7a74854f99a86f5d03d06939ea as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/8dbe2c7a74854f99a86f5d03d06939ea 2024-11-10T15:18:53,700 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/8dbe2c7a74854f99a86f5d03d06939ea, entries=150, sequenceid=62, filesize=11.7 K 2024-11-10T15:18:53,701 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for f49fc5144c71cae6bffc0a2f5b02ce84 in 498ms, sequenceid=62, compaction requested=true 2024-11-10T15:18:53,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:18:53,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:53,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-10T15:18:53,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-10T15:18:53,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-10T15:18:53,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 653 msec 2024-11-10T15:18:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:53,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731251933374, max timestamp=1731251933704Num uniques 1068; 2024-11-10T15:18:53,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-10T15:18:53,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731251933374, max timestamp=1731251933704Num uniques 1068; 2024-11-10T15:18:53,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-10T15:18:53,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731251933374, max timestamp=1731251933704Num uniques 1068; 2024-11-10T15:18:53,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-10T15:18:53,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=40.25 KB heapSize=105.23 KB 2024-11-10T15:18:53,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:18:53,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:53,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:18:53,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:53,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:18:53,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:53,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 656 msec 2024-11-10T15:18:53,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111027ddc071aa16495aae515be5f255bb9c_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251933706/Put/seqid=0 2024-11-10T15:18:53,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251933729, max timestamp=1731251933736Num uniques 859; 2024-11-10T15:18:53,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:18:53,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251933729, max timestamp=1731251933736Num uniques 859; 2024-11-10T15:18:53,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:18:53,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251933729, max timestamp=1731251933736Num uniques 859; 2024-11-10T15:18:53,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:18:53,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731251933782, max timestamp=1731251933794Num uniques 1068; 2024-11-10T15:18:53,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-10T15:18:53,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731251933782, max timestamp=1731251933794Num uniques 1068; 2024-11-10T15:18:53,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731251933782, max timestamp=1731251933794Num uniques 1068; 2024-11-10T15:18:53,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-10T15:18:53,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-10T15:18:53,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742421_1597 (size=19474) 2024-11-10T15:18:53,818 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:53,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251993815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251993817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251993818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251993821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251993822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,829 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111027ddc071aa16495aae515be5f255bb9c_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111027ddc071aa16495aae515be5f255bb9c_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:53,830 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/537ad95340ce4e5cac6f40711c67e3de, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:53,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/537ad95340ce4e5cac6f40711c67e3de is 175, key is test_row_0/A:col10/1731251933706/Put/seqid=0 2024-11-10T15:18:53,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742422_1598 (size=56733) 2024-11-10T15:18:53,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251993931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251993931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251993931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251993931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:53,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251993931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:54,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251994141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:54,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251994141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251994141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251994141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251994141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T15:18:54,191 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:54,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-10T15:18:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T15:18:54,193 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:54,194 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:54,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:54,234 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=102, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/537ad95340ce4e5cac6f40711c67e3de 2024-11-10T15:18:54,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/7688a5044b6540b8a557e1055f5c923f is 50, key is test_row_0/B:col10/1731251933706/Put/seqid=0 2024-11-10T15:18:54,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742423_1599 (size=12001) 2024-11-10T15:18:54,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/7688a5044b6540b8a557e1055f5c923f 2024-11-10T15:18:54,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/e61691bcab434acba7aab0cca54a0ad4 is 50, key is test_row_0/C:col10/1731251933706/Put/seqid=0 2024-11-10T15:18:54,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742424_1600 (size=12001) 2024-11-10T15:18:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T15:18:54,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T15:18:54,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:54,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:54,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:54,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:54,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:54,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251994452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:54,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251994452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:54,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251994453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:54,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251994453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:54,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251994454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:54,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:54,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:54,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:54,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T15:18:54,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:54,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:54,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:54,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:54,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T15:18:54,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T15:18:54,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:54,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:54,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:54,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:54,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:54,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/e61691bcab434acba7aab0cca54a0ad4 2024-11-10T15:18:54,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/537ad95340ce4e5cac6f40711c67e3de as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/537ad95340ce4e5cac6f40711c67e3de 2024-11-10T15:18:54,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/537ad95340ce4e5cac6f40711c67e3de, entries=300, sequenceid=102, filesize=55.4 K 2024-11-10T15:18:54,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/7688a5044b6540b8a557e1055f5c923f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/7688a5044b6540b8a557e1055f5c923f 2024-11-10T15:18:54,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/7688a5044b6540b8a557e1055f5c923f, entries=150, sequenceid=102, filesize=11.7 K 2024-11-10T15:18:54,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/e61691bcab434acba7aab0cca54a0ad4 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/e61691bcab434acba7aab0cca54a0ad4 2024-11-10T15:18:54,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/e61691bcab434acba7aab0cca54a0ad4, entries=150, sequenceid=102, filesize=11.7 K 2024-11-10T15:18:54,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~105.19 KB/107712, currentSize=147.60 KB/151140 for f49fc5144c71cae6bffc0a2f5b02ce84 in 1015ms, sequenceid=102, compaction requested=true 2024-11-10T15:18:54,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:18:54,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:54,723 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:18:54,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:54,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:54,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:54,723 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:18:54,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:54,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:54,724 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:18:54,724 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/A is initiating minor compaction (all files) 2024-11-10T15:18:54,724 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:18:54,724 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/A in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:54,724 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/B is initiating minor compaction (all files) 2024-11-10T15:18:54,724 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/B in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:54,725 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/a293088abd0d408881a57626a2f53dbe, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/85da13d91f554d62b066109173de538b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/b7eb39191ea44698b97d0489a0626a9f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/537ad95340ce4e5cac6f40711c67e3de] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=146.1 K 2024-11-10T15:18:54,725 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0bd2d43c5ec842aa8234b61116a6acf6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/affec116f2cb48fba7e02af1affc1189, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0a43b027dc604fb29339e5fc82ffdd51, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/7688a5044b6540b8a557e1055f5c923f] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=46.9 K 2024-11-10T15:18:54,725 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:54,725 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/a293088abd0d408881a57626a2f53dbe, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/85da13d91f554d62b066109173de538b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/b7eb39191ea44698b97d0489a0626a9f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/537ad95340ce4e5cac6f40711c67e3de] 2024-11-10T15:18:54,725 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a293088abd0d408881a57626a2f53dbe, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731251930898 2024-11-10T15:18:54,725 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bd2d43c5ec842aa8234b61116a6acf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731251930898 2024-11-10T15:18:54,725 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85da13d91f554d62b066109173de538b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731251931027 2024-11-10T15:18:54,725 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting affec116f2cb48fba7e02af1affc1189, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731251931027 2024-11-10T15:18:54,726 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7eb39191ea44698b97d0489a0626a9f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731251932183 2024-11-10T15:18:54,726 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 537ad95340ce4e5cac6f40711c67e3de, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731251933374 2024-11-10T15:18:54,726 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a43b027dc604fb29339e5fc82ffdd51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731251932183 2024-11-10T15:18:54,728 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7688a5044b6540b8a557e1055f5c923f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731251933374 2024-11-10T15:18:54,745 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:54,755 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#B#compaction#508 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:54,756 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/4631fd9233304d8eb808e257874e3f41 is 50, key is test_row_0/B:col10/1731251933706/Put/seqid=0 2024-11-10T15:18:54,760 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110d2cd222f448648a68a6f4e2cee2990fc_f49fc5144c71cae6bffc0a2f5b02ce84 store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:54,763 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110d2cd222f448648a68a6f4e2cee2990fc_f49fc5144c71cae6bffc0a2f5b02ce84, store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:54,763 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d2cd222f448648a68a6f4e2cee2990fc_f49fc5144c71cae6bffc0a2f5b02ce84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:54,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T15:18:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:54,808 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=147.60 KB heapSize=409.92 KB 2024-11-10T15:18:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:18:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:18:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:18:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:54,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742426_1602 (size=12139) 2024-11-10T15:18:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T15:18:54,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742425_1601 (size=4469) 2024-11-10T15:18:54,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109ad0e20388a2425f91fd0c5ca361893a_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251933821/Put/seqid=0 2024-11-10T15:18:54,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742427_1603 (size=12304) 2024-11-10T15:18:54,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:54,882 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109ad0e20388a2425f91fd0c5ca361893a_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109ad0e20388a2425f91fd0c5ca361893a_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:54,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/6f9179d912d34620975c877453870a7a, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:54,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/6f9179d912d34620975c877453870a7a is 175, key is test_row_0/A:col10/1731251933821/Put/seqid=0 2024-11-10T15:18:54,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742428_1604 (size=31105) 2024-11-10T15:18:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:54,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:54,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251994971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:54,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251994971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:54,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251994971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:54,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:54,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251994972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:54,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:54,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:54,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251994972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:54,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:54,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:55,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251995081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:55,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251995081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251995081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:55,223 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/4631fd9233304d8eb808e257874e3f41 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/4631fd9233304d8eb808e257874e3f41 2024-11-10T15:18:55,231 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/B of f49fc5144c71cae6bffc0a2f5b02ce84 into 4631fd9233304d8eb808e257874e3f41(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:55,231 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:18:55,231 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/B, priority=12, startTime=1731251934723; duration=0sec 2024-11-10T15:18:55,231 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:55,231 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:B 2024-11-10T15:18:55,231 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:18:55,232 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:18:55,232 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/C is initiating minor compaction (all files) 2024-11-10T15:18:55,232 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/C in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:55,232 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c08eb21eb71d4d6d8f579fb7b778d887, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ca834afc4b4e4a7984df068a6d2d7fdf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/8dbe2c7a74854f99a86f5d03d06939ea, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/e61691bcab434acba7aab0cca54a0ad4] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=46.9 K 2024-11-10T15:18:55,233 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting c08eb21eb71d4d6d8f579fb7b778d887, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731251930898 2024-11-10T15:18:55,233 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting ca834afc4b4e4a7984df068a6d2d7fdf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731251931027 2024-11-10T15:18:55,233 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dbe2c7a74854f99a86f5d03d06939ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731251932183 2024-11-10T15:18:55,233 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting e61691bcab434acba7aab0cca54a0ad4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731251933374 2024-11-10T15:18:55,248 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#A#compaction#507 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:55,249 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/0aa755e7318c4fe1a38b440040d0b56b is 175, key is test_row_0/A:col10/1731251933706/Put/seqid=0 2024-11-10T15:18:55,255 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#C#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:55,255 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/4d28de341bc54fa4a9d3dc96dedc40a4 is 50, key is test_row_0/C:col10/1731251933706/Put/seqid=0 2024-11-10T15:18:55,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742430_1606 (size=12139) 2024-11-10T15:18:55,294 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/6f9179d912d34620975c877453870a7a 2024-11-10T15:18:55,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251995293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251995293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:55,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:55,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251995293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:55,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:55,305 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/4d28de341bc54fa4a9d3dc96dedc40a4 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/4d28de341bc54fa4a9d3dc96dedc40a4 2024-11-10T15:18:55,313 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/C of f49fc5144c71cae6bffc0a2f5b02ce84 into 4d28de341bc54fa4a9d3dc96dedc40a4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:55,313 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:18:55,313 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/C, priority=12, startTime=1731251934723; duration=0sec 2024-11-10T15:18:55,313 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:55,313 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:C 2024-11-10T15:18:55,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742429_1605 (size=31093) 2024-11-10T15:18:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T15:18:55,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/78cc79439f614244a883baa341bcd36a is 50, key is test_row_0/B:col10/1731251933821/Put/seqid=0 2024-11-10T15:18:55,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742431_1607 (size=12151) 2024-11-10T15:18:55,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251995602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:55,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:55,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251995611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:55,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251995611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:55,721 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/0aa755e7318c4fe1a38b440040d0b56b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/0aa755e7318c4fe1a38b440040d0b56b 2024-11-10T15:18:55,724 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/A of f49fc5144c71cae6bffc0a2f5b02ce84 into 0aa755e7318c4fe1a38b440040d0b56b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-10T15:18:55,724 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:18:55,724 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/A, priority=12, startTime=1731251934723; duration=1sec 2024-11-10T15:18:55,724 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:55,724 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:A 2024-11-10T15:18:55,750 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/78cc79439f614244a883baa341bcd36a 2024-11-10T15:18:55,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/3451c060438846ceadff72caa442be6f is 50, key is test_row_0/C:col10/1731251933821/Put/seqid=0 2024-11-10T15:18:55,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742432_1608 (size=12151) 2024-11-10T15:18:55,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251995991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:55,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251995992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:55,993 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_2' in f49fc5144c71cae6bffc0a2f5b02ce84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:18:55,993 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in f49fc5144c71cae6bffc0a2f5b02ce84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:18:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:56,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251996111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:56,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251996121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:56,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251996121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:56,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:56,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:56,164 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/3451c060438846ceadff72caa442be6f 2024-11-10T15:18:56,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/6f9179d912d34620975c877453870a7a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/6f9179d912d34620975c877453870a7a 2024-11-10T15:18:56,176 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/6f9179d912d34620975c877453870a7a, entries=150, sequenceid=155, filesize=30.4 K 2024-11-10T15:18:56,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/78cc79439f614244a883baa341bcd36a as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/78cc79439f614244a883baa341bcd36a 2024-11-10T15:18:56,195 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/78cc79439f614244a883baa341bcd36a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-10T15:18:56,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/3451c060438846ceadff72caa442be6f as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/3451c060438846ceadff72caa442be6f 2024-11-10T15:18:56,204 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/3451c060438846ceadff72caa442be6f, entries=150, sequenceid=155, filesize=11.9 K 2024-11-10T15:18:56,205 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~409.88 KB/419712, currentSize=40.25 KB/41220 for f49fc5144c71cae6bffc0a2f5b02ce84 in 1397ms, sequenceid=155, compaction requested=false 2024-11-10T15:18:56,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:18:56,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:56,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-10T15:18:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-10T15:18:56,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-10T15:18:56,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0120 sec 2024-11-10T15:18:56,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.0160 sec 2024-11-10T15:18:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T15:18:56,341 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:56,342 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-10T15:18:56,344 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T15:18:56,345 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:56,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T15:18:56,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T15:18:56,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:56,497 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-10T15:18:56,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:18:56,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:56,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:18:56,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:56,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:18:56,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:56,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106ea34fcaece943e9979e3adf7547a4c1_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251934970/Put/seqid=0 2024-11-10T15:18:56,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742433_1609 (size=12304) 2024-11-10T15:18:56,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:56,527 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106ea34fcaece943e9979e3adf7547a4c1_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106ea34fcaece943e9979e3adf7547a4c1_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:56,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/358b9cb654cc42e09c48422147c85516, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:56,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/358b9cb654cc42e09c48422147c85516 is 175, key is test_row_0/A:col10/1731251934970/Put/seqid=0 2024-11-10T15:18:56,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742434_1610 (size=31105) 2024-11-10T15:18:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T15:18:56,934 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/358b9cb654cc42e09c48422147c85516 2024-11-10T15:18:56,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/aed6f15fa5554f91a356ab973816dec2 is 50, key is test_row_0/B:col10/1731251934970/Put/seqid=0 2024-11-10T15:18:56,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742435_1611 (size=12151) 2024-11-10T15:18:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T15:18:57,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:57,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731251937141, max timestamp=1731251937160Num uniques 859; 2024-11-10T15:18:57,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T15:18:57,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731251937141, max timestamp=1731251937160Num uniques 859; 2024-11-10T15:18:57,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T15:18:57,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731251937141, max timestamp=1731251937160Num uniques 859; 2024-11-10T15:18:57,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T15:18:57,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251997205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:57,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251997211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:57,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251997212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:57,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:57,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251997322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:57,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251997322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:57,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251997322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:57,359 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/aed6f15fa5554f91a356ab973816dec2 2024-11-10T15:18:57,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/92be9f494cee4085bad191b51f8078c2 is 50, key is test_row_0/C:col10/1731251934970/Put/seqid=0 2024-11-10T15:18:57,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742436_1612 (size=12151) 2024-11-10T15:18:57,374 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/92be9f494cee4085bad191b51f8078c2 2024-11-10T15:18:57,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/358b9cb654cc42e09c48422147c85516 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/358b9cb654cc42e09c48422147c85516 2024-11-10T15:18:57,380 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/358b9cb654cc42e09c48422147c85516, entries=150, sequenceid=167, filesize=30.4 K 2024-11-10T15:18:57,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/aed6f15fa5554f91a356ab973816dec2 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/aed6f15fa5554f91a356ab973816dec2 2024-11-10T15:18:57,383 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/aed6f15fa5554f91a356ab973816dec2, entries=150, sequenceid=167, filesize=11.9 K 2024-11-10T15:18:57,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/92be9f494cee4085bad191b51f8078c2 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/92be9f494cee4085bad191b51f8078c2 2024-11-10T15:18:57,387 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/92be9f494cee4085bad191b51f8078c2, entries=150, sequenceid=167, filesize=11.9 K 2024-11-10T15:18:57,388 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for f49fc5144c71cae6bffc0a2f5b02ce84 in 891ms, sequenceid=167, compaction requested=true 2024-11-10T15:18:57,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:18:57,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:57,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-10T15:18:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-10T15:18:57,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-10T15:18:57,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0440 sec 2024-11-10T15:18:57,393 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.0500 sec 2024-11-10T15:18:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T15:18:57,481 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:57,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-10T15:18:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T15:18:57,483 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:57,484 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:57,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:57,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-10T15:18:57,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:18:57,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:57,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:18:57,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:57,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:18:57,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:57,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251997545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:57,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251997546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:57,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251997547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:57,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:57,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:57,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:57,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d2f163c521c34ffdb14977442c8aeaf2_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251937200/Put/seqid=0 2024-11-10T15:18:57,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742437_1613 (size=14794) 2024-11-10T15:18:57,573 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,577 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d2f163c521c34ffdb14977442c8aeaf2_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d2f163c521c34ffdb14977442c8aeaf2_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:57,577 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/dba908803bcb4f2597e713fd520e8e92, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:57,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/dba908803bcb4f2597e713fd520e8e92 is 175, key is test_row_0/A:col10/1731251937200/Put/seqid=0 2024-11-10T15:18:57,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742438_1614 (size=39749) 2024-11-10T15:18:57,589 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/dba908803bcb4f2597e713fd520e8e92 2024-11-10T15:18:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T15:18:57,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/5ae9fc3d01e14f8cb31c8f539b148053 is 50, key is test_row_0/B:col10/1731251937200/Put/seqid=0 2024-11-10T15:18:57,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742439_1615 (size=12151) 2024-11-10T15:18:57,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/5ae9fc3d01e14f8cb31c8f539b148053 2024-11-10T15:18:57,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T15:18:57,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:57,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:57,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:57,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:57,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:57,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/90b980b1b9f844b78dc60e0943f29725 is 50, key is test_row_0/C:col10/1731251937200/Put/seqid=0 2024-11-10T15:18:57,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251997663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:57,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251997663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:57,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251997663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:57,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742440_1616 (size=12151) 2024-11-10T15:18:57,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/90b980b1b9f844b78dc60e0943f29725 2024-11-10T15:18:57,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/dba908803bcb4f2597e713fd520e8e92 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/dba908803bcb4f2597e713fd520e8e92 2024-11-10T15:18:57,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/dba908803bcb4f2597e713fd520e8e92, entries=200, sequenceid=204, filesize=38.8 K 2024-11-10T15:18:57,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/5ae9fc3d01e14f8cb31c8f539b148053 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/5ae9fc3d01e14f8cb31c8f539b148053 2024-11-10T15:18:57,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/5ae9fc3d01e14f8cb31c8f539b148053, entries=150, sequenceid=204, filesize=11.9 K 2024-11-10T15:18:57,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/90b980b1b9f844b78dc60e0943f29725 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/90b980b1b9f844b78dc60e0943f29725 2024-11-10T15:18:57,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/90b980b1b9f844b78dc60e0943f29725, entries=150, sequenceid=204, filesize=11.9 K 2024-11-10T15:18:57,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for f49fc5144c71cae6bffc0a2f5b02ce84 in 176ms, sequenceid=204, compaction requested=true 2024-11-10T15:18:57,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:18:57,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:18:57,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:57,712 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:18:57,712 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:18:57,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:18:57,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:57,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:18:57,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:57,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,715 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:18:57,715 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133052 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:18:57,715 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/A is initiating minor compaction (all files) 2024-11-10T15:18:57,715 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/B is initiating minor compaction (all files) 2024-11-10T15:18:57,715 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/A in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:57,715 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/0aa755e7318c4fe1a38b440040d0b56b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/6f9179d912d34620975c877453870a7a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/358b9cb654cc42e09c48422147c85516, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/dba908803bcb4f2597e713fd520e8e92] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=129.9 K 2024-11-10T15:18:57,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,715 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:57,715 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/0aa755e7318c4fe1a38b440040d0b56b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/6f9179d912d34620975c877453870a7a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/358b9cb654cc42e09c48422147c85516, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/dba908803bcb4f2597e713fd520e8e92] 2024-11-10T15:18:57,715 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/B in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:57,715 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/4631fd9233304d8eb808e257874e3f41, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/78cc79439f614244a883baa341bcd36a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/aed6f15fa5554f91a356ab973816dec2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/5ae9fc3d01e14f8cb31c8f539b148053] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=47.5 K 2024-11-10T15:18:57,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,715 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4631fd9233304d8eb808e257874e3f41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731251933374 2024-11-10T15:18:57,715 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aa755e7318c4fe1a38b440040d0b56b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731251933374 2024-11-10T15:18:57,716 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f9179d912d34620975c877453870a7a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731251933815 2024-11-10T15:18:57,716 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 78cc79439f614244a883baa341bcd36a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731251933815 2024-11-10T15:18:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,716 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting aed6f15fa5554f91a356ab973816dec2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731251934964 2024-11-10T15:18:57,716 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 358b9cb654cc42e09c48422147c85516, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731251934964 2024-11-10T15:18:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,716 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting dba908803bcb4f2597e713fd520e8e92, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731251937200 2024-11-10T15:18:57,716 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ae9fc3d01e14f8cb31c8f539b148053, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731251937200 2024-11-10T15:18:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,724 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,727 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#B#compaction#520 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,728 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/62846ad2427b4711a4309a7eb4a1a238 is 50, key is test_row_0/B:col10/1731251937200/Put/seqid=0 2024-11-10T15:18:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,731 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110747818b23b0947a78dadc43e597bf420_f49fc5144c71cae6bffc0a2f5b02ce84 store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,734 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110747818b23b0947a78dadc43e597bf420_f49fc5144c71cae6bffc0a2f5b02ce84, store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:57,734 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110747818b23b0947a78dadc43e597bf420_f49fc5144c71cae6bffc0a2f5b02ce84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:57,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742441_1617 (size=12425) 2024-11-10T15:18:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742442_1618 (size=4469) 2024-11-10T15:18:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,779 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/62846ad2427b4711a4309a7eb4a1a238 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/62846ad2427b4711a4309a7eb4a1a238 2024-11-10T15:18:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,784 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/B of f49fc5144c71cae6bffc0a2f5b02ce84 into 62846ad2427b4711a4309a7eb4a1a238(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:57,784 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:18:57,784 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/B, priority=12, startTime=1731251937712; duration=0sec 2024-11-10T15:18:57,784 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:18:57,784 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:B 2024-11-10T15:18:57,784 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:18:57,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,785 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:18:57,785 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/C is initiating minor compaction (all files) 2024-11-10T15:18:57,785 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/C in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:57,785 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/4d28de341bc54fa4a9d3dc96dedc40a4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/3451c060438846ceadff72caa442be6f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/92be9f494cee4085bad191b51f8078c2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/90b980b1b9f844b78dc60e0943f29725] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=47.5 K 2024-11-10T15:18:57,785 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d28de341bc54fa4a9d3dc96dedc40a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731251933374 2024-11-10T15:18:57,785 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3451c060438846ceadff72caa442be6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731251933815 2024-11-10T15:18:57,786 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 92be9f494cee4085bad191b51f8078c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731251934964 2024-11-10T15:18:57,786 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 90b980b1b9f844b78dc60e0943f29725, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731251937200 2024-11-10T15:18:57,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T15:18:57,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:57,792 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T15:18:57,793 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#C#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:57,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:18:57,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:57,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:18:57,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:57,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:18:57,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:18:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,793 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/9fb76be7930a4a3f89fd0d7f71134947 is 50, key is test_row_0/C:col10/1731251937200/Put/seqid=0 2024-11-10T15:18:57,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T15:18:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742443_1619 (size=12425) 2024-11-10T15:18:57,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106d49f4ba66b74aaeb60097632d77ba6b_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251937543/Put/seqid=0 2024-11-10T15:18:57,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,828 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/9fb76be7930a4a3f89fd0d7f71134947 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/9fb76be7930a4a3f89fd0d7f71134947 2024-11-10T15:18:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,834 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/C of f49fc5144c71cae6bffc0a2f5b02ce84 into 9fb76be7930a4a3f89fd0d7f71134947(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:57,834 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:18:57,834 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/C, priority=12, startTime=1731251937713; duration=0sec 2024-11-10T15:18:57,834 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:57,834 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:C 2024-11-10T15:18:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742444_1620 (size=9814) 2024-11-10T15:18:57,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:57,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:57,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731251937904, max timestamp=1731251937923Num uniques 859; 2024-11-10T15:18:57,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T15:18:57,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731251937904, max timestamp=1731251937923Num uniques 859; 2024-11-10T15:18:57,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T15:18:57,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731251937904, max timestamp=1731251937923Num uniques 859; 2024-11-10T15:18:57,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T15:18:57,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731251937958, max timestamp=1731251937982Num uniques 1118; 2024-11-10T15:18:57,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-10T15:18:57,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731251937958, max timestamp=1731251937982Num uniques 1118; 2024-11-10T15:18:57,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-10T15:18:58,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731251937958, max timestamp=1731251937982Num uniques 1118; 2024-11-10T15:18:58,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-10T15:18:58,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731251938015, max timestamp=1731251938038Num uniques 1127; 2024-11-10T15:18:58,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=14972, suffix heap size=145176,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=327 2024-11-10T15:18:58,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731251938018, max timestamp=1731251938038Num uniques 1027; 2024-11-10T15:18:58,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-10T15:18:58,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731251938018, max timestamp=1731251938038Num uniques 1027; 2024-11-10T15:18:58,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-10T15:18:58,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251998076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251998078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251998080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731251938060, max timestamp=1731251938086Num uniques 1186; 2024-11-10T15:18:58,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731251938060, max timestamp=1731251938086Num uniques 1036; 2024-11-10T15:18:58,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=10806, suffix heap size=151312,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=236 2024-11-10T15:18:58,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-10T15:18:58,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731251938060, max timestamp=1731251938086Num uniques 1036; 2024-11-10T15:18:58,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-11-10T15:18:58,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T15:18:58,185 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#A#compaction#519 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:18:58,185 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/86dae5e709844e6b8c81d0ea6afe4068 is 175, key is test_row_0/A:col10/1731251937200/Put/seqid=0 2024-11-10T15:18:58,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731251938167, max timestamp=1731251938198Num uniques 1045; 2024-11-10T15:18:58,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731251938167, max timestamp=1731251938198Num uniques 1095; 2024-11-10T15:18:58,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-10T15:18:58,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-10T15:18:58,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731251938167, max timestamp=1731251938198Num uniques 1009; 2024-11-10T15:18:58,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=11220, suffix heap size=132904,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=245 2024-11-10T15:18:58,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742445_1621 (size=31379) 2024-11-10T15:18:58,232 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/86dae5e709844e6b8c81d0ea6afe4068 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/86dae5e709844e6b8c81d0ea6afe4068 2024-11-10T15:18:58,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:58,242 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/A of f49fc5144c71cae6bffc0a2f5b02ce84 into 86dae5e709844e6b8c81d0ea6afe4068(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:18:58,242 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:18:58,242 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/A, priority=12, startTime=1731251937712; duration=0sec 2024-11-10T15:18:58,242 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:18:58,242 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:A 2024-11-10T15:18:58,244 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106d49f4ba66b74aaeb60097632d77ba6b_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106d49f4ba66b74aaeb60097632d77ba6b_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:58,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/06a3e2679f9e4fcc8deee656d6564a13, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:58,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/06a3e2679f9e4fcc8deee656d6564a13 is 175, key is test_row_0/A:col10/1731251937543/Put/seqid=0 2024-11-10T15:18:58,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742446_1622 (size=22461) 2024-11-10T15:18:58,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251998286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251998286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251998293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251998322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251998328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251998432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251998432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251998432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251998441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251998441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T15:18:58,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251998641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251998642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251998651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251998651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251998651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,717 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/06a3e2679f9e4fcc8deee656d6564a13 2024-11-10T15:18:58,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/5698436e4b6c4113bb53c85b70e0faa6 is 50, key is test_row_0/B:col10/1731251937543/Put/seqid=0 2024-11-10T15:18:58,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742447_1623 (size=9757) 2024-11-10T15:18:58,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251998952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251998952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731251998962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251998962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:58,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251998962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:58,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:58,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:58,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:59,126 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/5698436e4b6c4113bb53c85b70e0faa6 2024-11-10T15:18:59,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/46c6faf3560440b28f09b381ca9c16ea is 50, key is test_row_0/C:col10/1731251937543/Put/seqid=0 2024-11-10T15:18:59,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742448_1624 (size=9757) 2024-11-10T15:18:59,160 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/46c6faf3560440b28f09b381ca9c16ea 2024-11-10T15:18:59,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/06a3e2679f9e4fcc8deee656d6564a13 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/06a3e2679f9e4fcc8deee656d6564a13 2024-11-10T15:18:59,167 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/06a3e2679f9e4fcc8deee656d6564a13, entries=100, sequenceid=213, filesize=21.9 K 2024-11-10T15:18:59,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/5698436e4b6c4113bb53c85b70e0faa6 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/5698436e4b6c4113bb53c85b70e0faa6 2024-11-10T15:18:59,170 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/5698436e4b6c4113bb53c85b70e0faa6, entries=100, sequenceid=213, filesize=9.5 K 2024-11-10T15:18:59,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/46c6faf3560440b28f09b381ca9c16ea as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/46c6faf3560440b28f09b381ca9c16ea 2024-11-10T15:18:59,173 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/46c6faf3560440b28f09b381ca9c16ea, entries=100, sequenceid=213, filesize=9.5 K 2024-11-10T15:18:59,174 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for f49fc5144c71cae6bffc0a2f5b02ce84 in 1382ms, sequenceid=213, compaction requested=false 2024-11-10T15:18:59,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:18:59,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:59,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-10T15:18:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-10T15:18:59,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-10T15:18:59,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6910 sec 2024-11-10T15:18:59,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.6950 sec 2024-11-10T15:18:59,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=154.31 KB heapSize=437.39 KB 2024-11-10T15:18:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:59,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731251938286, max timestamp=1731251939463Num uniques 1104; 2024-11-10T15:18:59,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-10T15:18:59,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731251938286, max timestamp=1731251939463Num uniques 1104; 2024-11-10T15:18:59,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-10T15:18:59,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731251938286, max timestamp=1731251939463Num uniques 1104; 2024-11-10T15:18:59,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-10T15:18:59,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:18:59,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:59,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:18:59,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:59,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:18:59,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:18:59,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a8c397a58aa7445f8cdf6e432ee8ec34_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251938319/Put/seqid=0 2024-11-10T15:18:59,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742449_1625 (size=14994) 2024-11-10T15:18:59,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251939483, max timestamp=1731251939497Num uniques 859; 2024-11-10T15:18:59,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251939483, max timestamp=1731251939497Num uniques 859; 2024-11-10T15:18:59,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:18:59,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:18:59,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731251939483, max timestamp=1731251939497Num uniques 859; 2024-11-10T15:18:59,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T15:18:59,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731251939519, max timestamp=1731251939529Num uniques 1018; 2024-11-10T15:18:59,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T15:18:59,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731251939519, max timestamp=1731251939529Num uniques 1018; 2024-11-10T15:18:59,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731251939519, max timestamp=1731251939529Num uniques 1018; 2024-11-10T15:18:59,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-10T15:18:59,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T15:18:59,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731251939552, max timestamp=1731251939561Num uniques 1027; 2024-11-10T15:18:59,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-10T15:18:59,565 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731251939552, max timestamp=1731251939561Num uniques 1027; 2024-11-10T15:18:59,565 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-10T15:18:59,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731251939552, max timestamp=1731251939561Num uniques 1009; 2024-11-10T15:18:59,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=12682, suffix heap size=132904,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=277 2024-11-10T15:18:59,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T15:18:59,632 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:18:59,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:18:59,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-10T15:18:59,634 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:18:59,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T15:18:59,635 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:18:59,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:18:59,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:59,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:59,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251999638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:59,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731251999641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:59,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:59,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731251999642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:59,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731251939625, max timestamp=1731251939642Num uniques 1136; 2024-11-10T15:18:59,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-10T15:18:59,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731251939625, max timestamp=1731251939642Num uniques 1136; 2024-11-10T15:18:59,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-10T15:18:59,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731251939625, max timestamp=1731251939642Num uniques 1136; 2024-11-10T15:18:59,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-10T15:18:59,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:59,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:59,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:59,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731251939676, max timestamp=1731251939711Num uniques 1045; 2024-11-10T15:18:59,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-10T15:18:59,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731251939676, max timestamp=1731251939711Num uniques 1045; 2024-11-10T15:18:59,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-10T15:18:59,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731251939676, max timestamp=1731251939711Num uniques 1045; 2024-11-10T15:18:59,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-10T15:18:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T15:18:59,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T15:18:59,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:59,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:59,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:59,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:59,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:59,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251999782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:59,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:59,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:59,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731251939765, max timestamp=1731251939782Num uniques 1104; 2024-11-10T15:18:59,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731251999782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:59,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:59,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-10T15:18:59,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731251939765, max timestamp=1731251939782Num uniques 1104; 2024-11-10T15:18:59,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-10T15:18:59,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:59,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:59,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:59,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731251939775, max timestamp=1731251939782Num uniques 1104; 2024-11-10T15:18:59,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=6870, suffix heap size=142784,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=150 2024-11-10T15:18:59,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731251939854, max timestamp=1731251939868Num uniques 1063; 2024-11-10T15:18:59,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-10T15:18:59,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731251939854, max timestamp=1731251939868Num uniques 1063; 2024-11-10T15:18:59,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-10T15:18:59,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731251939854, max timestamp=1731251939868Num uniques 1009; 2024-11-10T15:18:59,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46208, new segment data size=7458, suffix heap size=132904,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=163 2024-11-10T15:18:59,890 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:18:59,899 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a8c397a58aa7445f8cdf6e432ee8ec34_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a8c397a58aa7445f8cdf6e432ee8ec34_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:18:59,899 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/d38e73c8968f42bbbfe81c8cf082e765, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:18:59,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/d38e73c8968f42bbbfe81c8cf082e765 is 175, key is test_row_0/A:col10/1731251938319/Put/seqid=0 2024-11-10T15:18:59,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731251939915, max timestamp=1731251939934Num uniques 1072; 2024-11-10T15:18:59,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-10T15:18:59,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731251939915, max timestamp=1731251939934Num uniques 1072; 2024-11-10T15:18:59,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-10T15:18:59,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731251939915, max timestamp=1731251939934Num uniques 1022; 2024-11-10T15:18:59,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-10T15:18:59,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742450_1626 (size=39949) 2024-11-10T15:18:59,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T15:18:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:18:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:18:59,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:18:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T15:18:59,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731251939957, max timestamp=1731251939965Num uniques 1081; 2024-11-10T15:18:59,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=12866, suffix heap size=140392,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=281 2024-11-10T15:18:59,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731251939957, max timestamp=1731251939965Num uniques 1081; 2024-11-10T15:18:59,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=12866, suffix heap size=140392,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=281 2024-11-10T15:18:59,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731251939957, max timestamp=1731251939965Num uniques 1081; 2024-11-10T15:18:59,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=12866, suffix heap size=140392,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=281 2024-11-10T15:18:59,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:18:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731251999988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:18:59,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:59,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:18:59,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:18:59,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731251939975, max timestamp=1731251939993Num uniques 1140; 2024-11-10T15:18:59,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731251939975, max timestamp=1731251939993Num uniques 1140; 2024-11-10T15:18:59,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-11-10T15:18:59,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-11-10T15:18:59,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731251939975, max timestamp=1731251939993Num uniques 1140; 2024-11-10T15:18:59,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-11-10T15:19:00,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731251940001, max timestamp=1731251940022Num uniques 1099; 2024-11-10T15:19:00,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-10T15:19:00,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731251940001, max timestamp=1731251940022Num uniques 1099; 2024-11-10T15:19:00,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-10T15:19:00,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731251940001, max timestamp=1731251940022Num uniques 1099; 2024-11-10T15:19:00,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-10T15:19:00,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1731251940051, max timestamp=1731251940079Num uniques 1108; 2024-11-10T15:19:00,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50738, new segment data size=9518, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-10T15:19:00,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1731251940051, max timestamp=1731251940079Num uniques 1108; 2024-11-10T15:19:00,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50738, new segment data size=9518, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-10T15:19:00,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731251940051, max timestamp=1731251940079Num uniques 1108; 2024-11-10T15:19:00,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-10T15:19:00,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T15:19:00,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:00,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:00,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:00,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1731251940097, max timestamp=1731251940115Num uniques 1067; 2024-11-10T15:19:00,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1731251940097, max timestamp=1731251940115Num uniques 1067; 2024-11-10T15:19:00,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=12222, suffix heap size=138936,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=267 2024-11-10T15:19:00,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=12222, suffix heap size=138936,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=267 2024-11-10T15:19:00,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731251940097, max timestamp=1731251940115Num uniques 1067; 2024-11-10T15:19:00,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=6870, suffix heap size=138936,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=150 2024-11-10T15:19:00,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731251940136, max timestamp=1731251940146Num uniques 1126; 2024-11-10T15:19:00,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=10346, suffix heap size=145072,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=226 2024-11-10T15:19:00,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731251940136, max timestamp=1731251940146Num uniques 1126; 2024-11-10T15:19:00,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=10346, suffix heap size=145072,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=226 2024-11-10T15:19:00,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731251940136, max timestamp=1731251940146Num uniques 1009; 2024-11-10T15:19:00,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10346, suffix heap size=132904,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=226 2024-11-10T15:19:00,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731251940165, max timestamp=1731251940188Num uniques 1085; 2024-11-10T15:19:00,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-11-10T15:19:00,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731251940165, max timestamp=1731251940188Num uniques 1085; 2024-11-10T15:19:00,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731251940165, max timestamp=1731251940188Num uniques 1085; 2024-11-10T15:19:00,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-11-10T15:19:00,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-11-10T15:19:00,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731251940201, max timestamp=1731251940222Num uniques 1094; 2024-11-10T15:19:00,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=13464, suffix heap size=141744,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=294 2024-11-10T15:19:00,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731251940201, max timestamp=1731251940222Num uniques 1094; 2024-11-10T15:19:00,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731251940201, max timestamp=1731251940222Num uniques 1094; 2024-11-10T15:19:00,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=13464, suffix heap size=141744,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=294 2024-11-10T15:19:00,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=13464, suffix heap size=141744,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=294 2024-11-10T15:19:00,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T15:19:00,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:00,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:00,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:00,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T15:19:00,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252000269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252000277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731252000281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252000284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731252000284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,342 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/d38e73c8968f42bbbfe81c8cf082e765 2024-11-10T15:19:00,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/0b5eaf62ab904fe2a479c5c953df1c25 is 50, key is test_row_0/B:col10/1731251938319/Put/seqid=0 2024-11-10T15:19:00,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742451_1627 (size=12301) 2024-11-10T15:19:00,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252000391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252000391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T15:19:00,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:00,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:00,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:00,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252000402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731252000402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731252000402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T15:19:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:00,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252000601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252000601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252000611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731252000612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731252000612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T15:19:00,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:00,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:00,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:00,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T15:19:00,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/0b5eaf62ab904fe2a479c5c953df1c25 2024-11-10T15:19:00,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/c1b4e3c8657f4a39a0c45a786db6223d is 50, key is test_row_0/C:col10/1731251938319/Put/seqid=0 2024-11-10T15:19:00,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742452_1628 (size=12301) 2024-11-10T15:19:00,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T15:19:00,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:00,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:00,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:00,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:00,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252000911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252000911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252000922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731252000922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731252000923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:01,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T15:19:01,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:01,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:01,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:01,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:01,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:01,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T15:19:01,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:01,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:01,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:01,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:01,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:01,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/c1b4e3c8657f4a39a0c45a786db6223d 2024-11-10T15:19:01,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/d38e73c8968f42bbbfe81c8cf082e765 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d38e73c8968f42bbbfe81c8cf082e765 2024-11-10T15:19:01,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d38e73c8968f42bbbfe81c8cf082e765, entries=200, sequenceid=323, filesize=39.0 K 2024-11-10T15:19:01,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/0b5eaf62ab904fe2a479c5c953df1c25 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0b5eaf62ab904fe2a479c5c953df1c25 2024-11-10T15:19:01,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0b5eaf62ab904fe2a479c5c953df1c25, entries=150, sequenceid=323, filesize=12.0 K 2024-11-10T15:19:01,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/c1b4e3c8657f4a39a0c45a786db6223d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c1b4e3c8657f4a39a0c45a786db6223d 2024-11-10T15:19:01,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c1b4e3c8657f4a39a0c45a786db6223d, entries=150, sequenceid=323, filesize=12.0 K 2024-11-10T15:19:01,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.58 KB/88656, currentSize=154.31 KB/158010 for f49fc5144c71cae6bffc0a2f5b02ce84 in 1767ms, sequenceid=323, compaction requested=true 2024-11-10T15:19:01,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:01,231 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:19:01,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:19:01,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:01,231 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:19:01,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:19:01,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:01,232 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:19:01,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:19:01,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:19:01,232 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/A is initiating minor compaction (all files) 2024-11-10T15:19:01,232 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/A in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:01,232 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/86dae5e709844e6b8c81d0ea6afe4068, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/06a3e2679f9e4fcc8deee656d6564a13, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d38e73c8968f42bbbfe81c8cf082e765] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=91.6 K 2024-11-10T15:19:01,232 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:01,232 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/86dae5e709844e6b8c81d0ea6afe4068, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/06a3e2679f9e4fcc8deee656d6564a13, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d38e73c8968f42bbbfe81c8cf082e765] 2024-11-10T15:19:01,232 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86dae5e709844e6b8c81d0ea6afe4068, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731251937200 2024-11-10T15:19:01,233 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06a3e2679f9e4fcc8deee656d6564a13, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731251937543 2024-11-10T15:19:01,233 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:19:01,233 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/B is initiating minor compaction (all files) 2024-11-10T15:19:01,233 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/B in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:01,233 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d38e73c8968f42bbbfe81c8cf082e765, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731251938286 2024-11-10T15:19:01,233 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/62846ad2427b4711a4309a7eb4a1a238, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/5698436e4b6c4113bb53c85b70e0faa6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0b5eaf62ab904fe2a479c5c953df1c25] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=33.7 K 2024-11-10T15:19:01,234 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 62846ad2427b4711a4309a7eb4a1a238, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731251937200 2024-11-10T15:19:01,234 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5698436e4b6c4113bb53c85b70e0faa6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731251937543 2024-11-10T15:19:01,234 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b5eaf62ab904fe2a479c5c953df1c25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731251938286 2024-11-10T15:19:01,247 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:01,248 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#B#compaction#528 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:19:01,249 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/6bae7f83489546be9cbf70fd64bae767 is 50, key is test_row_0/B:col10/1731251938319/Put/seqid=0 2024-11-10T15:19:01,250 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110c84ee04b3f204b829b13493ae22be98f_f49fc5144c71cae6bffc0a2f5b02ce84 store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:01,252 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110c84ee04b3f204b829b13493ae22be98f_f49fc5144c71cae6bffc0a2f5b02ce84, store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:01,252 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c84ee04b3f204b829b13493ae22be98f_f49fc5144c71cae6bffc0a2f5b02ce84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:01,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742453_1629 (size=12677) 2024-11-10T15:19:01,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742454_1630 (size=4469) 2024-11-10T15:19:01,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T15:19:01,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:01,321 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=154.31 KB heapSize=432.09 KB 2024-11-10T15:19:01,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:19:01,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:19:01,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:19:01,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:19:01,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:19:01,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:19:01,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b75a10c0ea434cca9c868ce7906b2190_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251940279/Put/seqid=0 2024-11-10T15:19:01,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742455_1631 (size=12454) 2024-11-10T15:19:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:01,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:01,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:01,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731252001432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731252001432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:01,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252001487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:01,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:01,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252001487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:01,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:01,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:01,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252001487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:01,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:01,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:01,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252001602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:01,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252001602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252001602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:01,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:01,663 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#A#compaction#529 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:19:01,663 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/3363cad0f8d44041940ebbdd53190989 is 175, key is test_row_0/A:col10/1731251938319/Put/seqid=0 2024-11-10T15:19:01,664 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/6bae7f83489546be9cbf70fd64bae767 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/6bae7f83489546be9cbf70fd64bae767 2024-11-10T15:19:01,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742456_1632 (size=31631) 2024-11-10T15:19:01,670 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/3363cad0f8d44041940ebbdd53190989 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/3363cad0f8d44041940ebbdd53190989 2024-11-10T15:19:01,671 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/B of f49fc5144c71cae6bffc0a2f5b02ce84 into 6bae7f83489546be9cbf70fd64bae767(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:19:01,671 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:01,671 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/B, priority=13, startTime=1731251941231; duration=0sec 2024-11-10T15:19:01,671 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:19:01,671 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:B 2024-11-10T15:19:01,671 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:19:01,672 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:19:01,672 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/C is initiating minor compaction (all files) 2024-11-10T15:19:01,672 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/C in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:01,672 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/9fb76be7930a4a3f89fd0d7f71134947, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/46c6faf3560440b28f09b381ca9c16ea, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c1b4e3c8657f4a39a0c45a786db6223d] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=33.7 K 2024-11-10T15:19:01,672 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fb76be7930a4a3f89fd0d7f71134947, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731251937200 2024-11-10T15:19:01,672 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 46c6faf3560440b28f09b381ca9c16ea, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731251937543 2024-11-10T15:19:01,672 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting c1b4e3c8657f4a39a0c45a786db6223d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731251938286 2024-11-10T15:19:01,674 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/A of f49fc5144c71cae6bffc0a2f5b02ce84 into 3363cad0f8d44041940ebbdd53190989(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:19:01,674 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:01,674 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/A, priority=13, startTime=1731251941231; duration=0sec 2024-11-10T15:19:01,674 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:01,674 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:A 2024-11-10T15:19:01,678 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#C#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:19:01,678 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/be7ac0dd5bb340b9a21b049eb12db20c is 50, key is test_row_0/C:col10/1731251938319/Put/seqid=0 2024-11-10T15:19:01,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742457_1633 (size=12677) 2024-11-10T15:19:01,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:01,745 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b75a10c0ea434cca9c868ce7906b2190_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b75a10c0ea434cca9c868ce7906b2190_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:01,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/cc8e8c75d16646f5ba09a0005cb7f863, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:01,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/cc8e8c75d16646f5ba09a0005cb7f863 is 175, key is test_row_0/A:col10/1731251940279/Put/seqid=0 2024-11-10T15:19:01,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742458_1634 (size=31255) 2024-11-10T15:19:01,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T15:19:01,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252001812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:01,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252001812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252001812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:01,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:01,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:02,088 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/be7ac0dd5bb340b9a21b049eb12db20c as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/be7ac0dd5bb340b9a21b049eb12db20c 2024-11-10T15:19:02,091 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/C of f49fc5144c71cae6bffc0a2f5b02ce84 into be7ac0dd5bb340b9a21b049eb12db20c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:19:02,091 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:02,091 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/C, priority=13, startTime=1731251941232; duration=0sec 2024-11-10T15:19:02,091 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:02,091 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:C 2024-11-10T15:19:02,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252002122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:02,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:02,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:02,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:02,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252002122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:02,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:02,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:02,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252002122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:02,150 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=618, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/cc8e8c75d16646f5ba09a0005cb7f863 2024-11-10T15:19:02,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/50181781beb04adfa96791da5d10de2e is 50, key is test_row_0/B:col10/1731251940279/Put/seqid=0 2024-11-10T15:19:02,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742459_1635 (size=12301) 2024-11-10T15:19:02,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:02,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731252002492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:02,493 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_1' in f49fc5144c71cae6bffc0a2f5b02ce84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:19:02,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:02,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:02,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:02,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:02,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731252002501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:02,503 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in f49fc5144c71cae6bffc0a2f5b02ce84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:19:02,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:02,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:02,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:02,562 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/50181781beb04adfa96791da5d10de2e 2024-11-10T15:19:02,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/6d6b34ce7e314b5584b39f7437925eba is 50, key is test_row_0/C:col10/1731251940279/Put/seqid=0 2024-11-10T15:19:02,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742460_1636 (size=12301) 2024-11-10T15:19:02,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252002631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:02,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:02,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252002631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252002631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:02,979 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/6d6b34ce7e314b5584b39f7437925eba 2024-11-10T15:19:02,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/cc8e8c75d16646f5ba09a0005cb7f863 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/cc8e8c75d16646f5ba09a0005cb7f863 2024-11-10T15:19:02,985 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/cc8e8c75d16646f5ba09a0005cb7f863, entries=150, sequenceid=618, filesize=30.5 K 2024-11-10T15:19:02,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/50181781beb04adfa96791da5d10de2e as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/50181781beb04adfa96791da5d10de2e 2024-11-10T15:19:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,988 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/50181781beb04adfa96791da5d10de2e, entries=150, sequenceid=618, filesize=12.0 K 2024-11-10T15:19:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/6d6b34ce7e314b5584b39f7437925eba as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/6d6b34ce7e314b5584b39f7437925eba 2024-11-10T15:19:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,991 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/6d6b34ce7e314b5584b39f7437925eba, entries=150, sequenceid=618, filesize=12.0 K 2024-11-10T15:19:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,992 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~432.05 KB/442416, currentSize=33.54 KB/34350 for f49fc5144c71cae6bffc0a2f5b02ce84 in 1671ms, sequenceid=618, compaction requested=false 2024-11-10T15:19:02,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:02,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-10T15:19:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-10T15:19:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-10T15:19:02,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3580 sec 2024-11-10T15:19:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 3.3630 sec 2024-11-10T15:19:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T15:19:03,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:19:03,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:03,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:19:03,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:03,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:19:03,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:03,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111033910b61afed47fda225bd0be53649ca_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251941433/Put/seqid=0 2024-11-10T15:19:03,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742461_1637 (size=14994) 2024-11-10T15:19:03,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251943721, max timestamp=1731251943728Num uniques 859; 2024-11-10T15:19:03,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251943721, max timestamp=1731251943728Num uniques 859; 2024-11-10T15:19:03,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:19:03,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:19:03,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251943721, max timestamp=1731251943728Num uniques 859; 2024-11-10T15:19:03,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:19:03,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252003757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:03,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252003757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:03,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252003763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T15:19:03,812 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:19:03,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:19:03,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-10T15:19:03,814 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:19:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T15:19:03,814 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:19:03,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:19:03,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252003871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:03,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252003872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:03,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252003872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T15:19:03,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T15:19:03,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:03,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:03,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:03,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:03,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:03,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,079 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:04,082 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111033910b61afed47fda225bd0be53649ca_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111033910b61afed47fda225bd0be53649ca_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:04,082 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/2345deed2aa8449d9c1b645261b77749, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:04,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/2345deed2aa8449d9c1b645261b77749 is 175, key is test_row_0/A:col10/1731251941433/Put/seqid=0 2024-11-10T15:19:04,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252004081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:04,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252004082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:04,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252004082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742462_1638 (size=39945) 2024-11-10T15:19:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:04,088 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=631, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/2345deed2aa8449d9c1b645261b77749 2024-11-10T15:19:04,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/f70ebf240f9846e8a1f7fc625c9234ec is 50, key is test_row_0/B:col10/1731251941433/Put/seqid=0 2024-11-10T15:19:04,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742463_1639 (size=9857) 2024-11-10T15:19:04,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T15:19:04,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:04,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:04,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:04,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T15:19:04,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T15:19:04,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:04,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:04,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:04,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252004391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252004391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252004391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:04,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T15:19:04,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:04,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:04,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:04,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T15:19:04,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/f70ebf240f9846e8a1f7fc625c9234ec 2024-11-10T15:19:04,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/8bfcf0d8c001411e83ac4775f7f035fc is 50, key is test_row_0/C:col10/1731251941433/Put/seqid=0 2024-11-10T15:19:04,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742464_1640 (size=9857) 2024-11-10T15:19:04,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731252004521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:04,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731252004522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:04,526 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_1' in f49fc5144c71cae6bffc0a2f5b02ce84 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:19:04,526 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in f49fc5144c71cae6bffc0a2f5b02ce84 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:19:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:04,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T15:19:04,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:04,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:04,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:04,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T15:19:04,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:04,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:04,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:04,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T15:19:04,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:04,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:04,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:04,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:04,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:04,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252004901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:04,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252004901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:04,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:04,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252004911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:04,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:04,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:04,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/8bfcf0d8c001411e83ac4775f7f035fc 2024-11-10T15:19:04,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/2345deed2aa8449d9c1b645261b77749 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/2345deed2aa8449d9c1b645261b77749 2024-11-10T15:19:04,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/2345deed2aa8449d9c1b645261b77749, entries=200, sequenceid=631, filesize=39.0 K 2024-11-10T15:19:04,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/f70ebf240f9846e8a1f7fc625c9234ec as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/f70ebf240f9846e8a1f7fc625c9234ec 2024-11-10T15:19:04,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/f70ebf240f9846e8a1f7fc625c9234ec, entries=100, sequenceid=631, filesize=9.6 K 2024-11-10T15:19:04,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/8bfcf0d8c001411e83ac4775f7f035fc as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/8bfcf0d8c001411e83ac4775f7f035fc 2024-11-10T15:19:04,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/8bfcf0d8c001411e83ac4775f7f035fc, entries=100, sequenceid=631, filesize=9.6 K 2024-11-10T15:19:04,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f49fc5144c71cae6bffc0a2f5b02ce84 in 1279ms, sequenceid=631, compaction requested=true 2024-11-10T15:19:04,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:04,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:19:04,932 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:19:04,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:04,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:19:04,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:04,933 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:19:04,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:19:04,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:19:04,933 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:19:04,933 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102831 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:19:04,933 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/B is initiating minor compaction (all files) 2024-11-10T15:19:04,933 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/A is initiating minor compaction (all files) 2024-11-10T15:19:04,933 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/B in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:04,933 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/A in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:04,933 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/6bae7f83489546be9cbf70fd64bae767, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/50181781beb04adfa96791da5d10de2e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/f70ebf240f9846e8a1f7fc625c9234ec] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=34.0 K 2024-11-10T15:19:04,933 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/3363cad0f8d44041940ebbdd53190989, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/cc8e8c75d16646f5ba09a0005cb7f863, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/2345deed2aa8449d9c1b645261b77749] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=100.4 K 2024-11-10T15:19:04,933 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:04,934 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/3363cad0f8d44041940ebbdd53190989, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/cc8e8c75d16646f5ba09a0005cb7f863, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/2345deed2aa8449d9c1b645261b77749] 2024-11-10T15:19:04,934 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bae7f83489546be9cbf70fd64bae767, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731251938286 2024-11-10T15:19:04,934 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3363cad0f8d44041940ebbdd53190989, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731251938286 2024-11-10T15:19:04,934 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 50181781beb04adfa96791da5d10de2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1731251940277 2024-11-10T15:19:04,934 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc8e8c75d16646f5ba09a0005cb7f863, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1731251940277 2024-11-10T15:19:04,934 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting f70ebf240f9846e8a1f7fc625c9234ec, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1731251941433 2024-11-10T15:19:04,934 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2345deed2aa8449d9c1b645261b77749, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1731251941433 2024-11-10T15:19:04,939 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:04,940 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#B#compaction#537 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:19:04,940 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/8ce17ffae3a94ad8a4442330bf2f8ca3 is 50, key is test_row_0/B:col10/1731251941433/Put/seqid=0 2024-11-10T15:19:04,942 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110d64ce019402b43e1a6d55aabcce30225_f49fc5144c71cae6bffc0a2f5b02ce84 store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:04,945 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110d64ce019402b43e1a6d55aabcce30225_f49fc5144c71cae6bffc0a2f5b02ce84, store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:04,945 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d64ce019402b43e1a6d55aabcce30225_f49fc5144c71cae6bffc0a2f5b02ce84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T15:19:04,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742466_1642 (size=4469) 2024-11-10T15:19:04,957 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#A#compaction#538 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:19:04,958 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/d504e4b2e2ac4e2eb397299693106030 is 175, key is test_row_0/A:col10/1731251941433/Put/seqid=0 2024-11-10T15:19:04,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742465_1641 (size=12779) 2024-11-10T15:19:04,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742467_1643 (size=31840) 2024-11-10T15:19:04,980 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/d504e4b2e2ac4e2eb397299693106030 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d504e4b2e2ac4e2eb397299693106030 2024-11-10T15:19:04,985 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/A of f49fc5144c71cae6bffc0a2f5b02ce84 into d504e4b2e2ac4e2eb397299693106030(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:19:04,985 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:04,985 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/A, priority=13, startTime=1731251944932; duration=0sec 2024-11-10T15:19:04,985 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:19:04,985 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:A 2024-11-10T15:19:04,985 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:19:04,986 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:19:04,986 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/C is initiating minor compaction (all files) 2024-11-10T15:19:04,986 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/C in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:04,986 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/be7ac0dd5bb340b9a21b049eb12db20c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/6d6b34ce7e314b5584b39f7437925eba, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/8bfcf0d8c001411e83ac4775f7f035fc] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=34.0 K 2024-11-10T15:19:04,986 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting be7ac0dd5bb340b9a21b049eb12db20c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731251938286 2024-11-10T15:19:04,987 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d6b34ce7e314b5584b39f7437925eba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1731251940277 2024-11-10T15:19:04,987 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bfcf0d8c001411e83ac4775f7f035fc, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1731251941433 2024-11-10T15:19:04,995 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#C#compaction#539 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:19:04,996 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/68b2eed51b5643fdaa349352fc9d2c04 is 50, key is test_row_0/C:col10/1731251941433/Put/seqid=0 2024-11-10T15:19:05,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742468_1644 (size=12779) 2024-11-10T15:19:05,018 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/68b2eed51b5643fdaa349352fc9d2c04 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/68b2eed51b5643fdaa349352fc9d2c04 2024-11-10T15:19:05,024 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/C of f49fc5144c71cae6bffc0a2f5b02ce84 into 68b2eed51b5643fdaa349352fc9d2c04(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:19:05,024 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:05,024 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/C, priority=13, startTime=1731251944933; duration=0sec 2024-11-10T15:19:05,024 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:05,025 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:C 2024-11-10T15:19:05,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T15:19:05,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:05,038 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-10T15:19:05,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:19:05,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:19:05,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:19:05,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:19:05,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:19:05,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:19:05,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e61dd8ed21ae455e8a9be255545db48b_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251943755/Put/seqid=0 2024-11-10T15:19:05,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742469_1645 (size=12454) 2024-11-10T15:19:05,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:05,092 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e61dd8ed21ae455e8a9be255545db48b_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e61dd8ed21ae455e8a9be255545db48b_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:05,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/6b7c94be3928461687213261ce158714, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:05,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/6b7c94be3928461687213261ce158714 is 175, key is test_row_0/A:col10/1731251943755/Put/seqid=0 2024-11-10T15:19:05,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742470_1646 (size=31255) 2024-11-10T15:19:05,371 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/8ce17ffae3a94ad8a4442330bf2f8ca3 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/8ce17ffae3a94ad8a4442330bf2f8ca3 2024-11-10T15:19:05,375 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/B of f49fc5144c71cae6bffc0a2f5b02ce84 into 8ce17ffae3a94ad8a4442330bf2f8ca3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:19:05,375 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:05,375 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/B, priority=13, startTime=1731251944933; duration=0sec 2024-11-10T15:19:05,375 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:05,375 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:B 2024-11-10T15:19:05,518 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=669, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/6b7c94be3928461687213261ce158714 2024-11-10T15:19:05,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/99ed91956e99445f98b72d46ec3afcb6 is 50, key is test_row_0/B:col10/1731251943755/Put/seqid=0 2024-11-10T15:19:05,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742471_1647 (size=12301) 2024-11-10T15:19:05,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:05,929 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/99ed91956e99445f98b72d46ec3afcb6 2024-11-10T15:19:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/c789dd6658d944a2ad87794e1324a2aa is 50, key is test_row_0/C:col10/1731251943755/Put/seqid=0 2024-11-10T15:19:05,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252005933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:05,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:05,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:05,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:05,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252005935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:05,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:05,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252005936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:05,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:05,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:05,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742472_1648 (size=12301) 2024-11-10T15:19:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T15:19:06,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252006042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:06,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252006042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:06,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252006042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:06,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:06,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252006252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:06,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252006252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252006252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:06,339 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/c789dd6658d944a2ad87794e1324a2aa 2024-11-10T15:19:06,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/6b7c94be3928461687213261ce158714 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/6b7c94be3928461687213261ce158714 2024-11-10T15:19:06,347 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/6b7c94be3928461687213261ce158714, entries=150, sequenceid=669, filesize=30.5 K 2024-11-10T15:19:06,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/99ed91956e99445f98b72d46ec3afcb6 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/99ed91956e99445f98b72d46ec3afcb6 2024-11-10T15:19:06,352 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/99ed91956e99445f98b72d46ec3afcb6, entries=150, sequenceid=669, filesize=12.0 K 2024-11-10T15:19:06,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/c789dd6658d944a2ad87794e1324a2aa as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c789dd6658d944a2ad87794e1324a2aa 2024-11-10T15:19:06,356 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c789dd6658d944a2ad87794e1324a2aa, entries=150, sequenceid=669, filesize=12.0 K 2024-11-10T15:19:06,357 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for f49fc5144c71cae6bffc0a2f5b02ce84 in 1318ms, sequenceid=669, compaction requested=false 2024-11-10T15:19:06,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:06,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:06,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-10T15:19:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-10T15:19:06,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-10T15:19:06,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5430 sec 2024-11-10T15:19:06,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.5500 sec 2024-11-10T15:19:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:06,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T15:19:06,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:19:06,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:06,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:19:06,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:06,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:19:06,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:06,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e2bf6c4099664f5f936eb4884351eafb_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251946563/Put/seqid=0 2024-11-10T15:19:06,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742473_1649 (size=14994) 2024-11-10T15:19:06,575 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:06,577 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e2bf6c4099664f5f936eb4884351eafb_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e2bf6c4099664f5f936eb4884351eafb_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:06,578 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/e2092beab7a74ff683e3c4595d73fcb7, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:06,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/e2092beab7a74ff683e3c4595d73fcb7 is 175, key is test_row_0/A:col10/1731251946563/Put/seqid=0 2024-11-10T15:19:06,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742474_1650 (size=39949) 2024-11-10T15:19:06,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252006597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:06,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252006599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:06,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:06,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252006600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:06,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252006711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:06,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252006712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:06,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:06,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:06,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252006712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:06,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252006921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:06,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:06,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252006921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252006921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:06,982 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=682, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/e2092beab7a74ff683e3c4595d73fcb7 2024-11-10T15:19:06,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/6b9ee732cb954ba0943a9e6349a67776 is 50, key is test_row_0/B:col10/1731251946563/Put/seqid=0 2024-11-10T15:19:06,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742475_1651 (size=12301) 2024-11-10T15:19:06,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/6b9ee732cb954ba0943a9e6349a67776 2024-11-10T15:19:06,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/ce48d2d2214c4fa186dbfe5b3cac4d42 is 50, key is test_row_0/C:col10/1731251946563/Put/seqid=0 2024-11-10T15:19:07,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742476_1652 (size=12301) 2024-11-10T15:19:07,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/ce48d2d2214c4fa186dbfe5b3cac4d42 2024-11-10T15:19:07,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/e2092beab7a74ff683e3c4595d73fcb7 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/e2092beab7a74ff683e3c4595d73fcb7 2024-11-10T15:19:07,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/e2092beab7a74ff683e3c4595d73fcb7, entries=200, sequenceid=682, filesize=39.0 K 2024-11-10T15:19:07,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/6b9ee732cb954ba0943a9e6349a67776 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/6b9ee732cb954ba0943a9e6349a67776 2024-11-10T15:19:07,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/6b9ee732cb954ba0943a9e6349a67776, entries=150, sequenceid=682, filesize=12.0 K 2024-11-10T15:19:07,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/ce48d2d2214c4fa186dbfe5b3cac4d42 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ce48d2d2214c4fa186dbfe5b3cac4d42 2024-11-10T15:19:07,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ce48d2d2214c4fa186dbfe5b3cac4d42, entries=150, sequenceid=682, filesize=12.0 K 2024-11-10T15:19:07,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f49fc5144c71cae6bffc0a2f5b02ce84 in 451ms, sequenceid=682, compaction requested=true 2024-11-10T15:19:07,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:07,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:19:07,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:07,014 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:19:07,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:19:07,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:07,015 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:19:07,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:19:07,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:19:07,015 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:19:07,015 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103044 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:19:07,015 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/B is initiating minor compaction (all files) 2024-11-10T15:19:07,015 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/A is initiating minor compaction (all files) 2024-11-10T15:19:07,015 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/A in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:07,015 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/B in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:07,015 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d504e4b2e2ac4e2eb397299693106030, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/6b7c94be3928461687213261ce158714, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/e2092beab7a74ff683e3c4595d73fcb7] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=100.6 K 2024-11-10T15:19:07,015 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/8ce17ffae3a94ad8a4442330bf2f8ca3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/99ed91956e99445f98b72d46ec3afcb6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/6b9ee732cb954ba0943a9e6349a67776] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=36.5 K 2024-11-10T15:19:07,015 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:07,015 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d504e4b2e2ac4e2eb397299693106030, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/6b7c94be3928461687213261ce158714, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/e2092beab7a74ff683e3c4595d73fcb7] 2024-11-10T15:19:07,016 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ce17ffae3a94ad8a4442330bf2f8ca3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1731251940280 2024-11-10T15:19:07,016 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d504e4b2e2ac4e2eb397299693106030, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1731251940280 2024-11-10T15:19:07,016 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 99ed91956e99445f98b72d46ec3afcb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1731251943753 2024-11-10T15:19:07,016 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b7c94be3928461687213261ce158714, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1731251943753 2024-11-10T15:19:07,016 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b9ee732cb954ba0943a9e6349a67776, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1731251945935 2024-11-10T15:19:07,016 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2092beab7a74ff683e3c4595d73fcb7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1731251945914 2024-11-10T15:19:07,020 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:07,022 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111079a78f10906e45ebbfbbe9656e7064b5_f49fc5144c71cae6bffc0a2f5b02ce84 store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:07,024 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#B#compaction#547 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:19:07,024 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/945cd1f5b8fa4c8b83ca8e09f4bf4457 is 50, key is test_row_0/B:col10/1731251946563/Put/seqid=0 2024-11-10T15:19:07,024 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111079a78f10906e45ebbfbbe9656e7064b5_f49fc5144c71cae6bffc0a2f5b02ce84, store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:07,024 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111079a78f10906e45ebbfbbe9656e7064b5_f49fc5144c71cae6bffc0a2f5b02ce84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:07,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742477_1653 (size=4469) 2024-11-10T15:19:07,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742478_1654 (size=12881) 2024-11-10T15:19:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:07,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T15:19:07,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:19:07,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:19:07,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:19:07,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:19:07,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:19:07,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:19:07,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111017c535ec8d2d4cd9a14810421e632a5d_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251946590/Put/seqid=0 2024-11-10T15:19:07,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742479_1655 (size=12454) 2024-11-10T15:19:07,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252007247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252007247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:07,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252007247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:07,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252007361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:07,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252007361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252007362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:07,431 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#A#compaction#546 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:19:07,432 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/9579f288214b4260b283f7630bcc4171 is 175, key is test_row_0/A:col10/1731251946563/Put/seqid=0 2024-11-10T15:19:07,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742480_1656 (size=31835) 2024-11-10T15:19:07,437 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/945cd1f5b8fa4c8b83ca8e09f4bf4457 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/945cd1f5b8fa4c8b83ca8e09f4bf4457 2024-11-10T15:19:07,440 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/B of f49fc5144c71cae6bffc0a2f5b02ce84 into 945cd1f5b8fa4c8b83ca8e09f4bf4457(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:19:07,440 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:07,440 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/B, priority=13, startTime=1731251947014; duration=0sec 2024-11-10T15:19:07,440 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:19:07,440 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:B 2024-11-10T15:19:07,440 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T15:19:07,441 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T15:19:07,441 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/C is initiating minor compaction (all files) 2024-11-10T15:19:07,441 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/C in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:07,441 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/68b2eed51b5643fdaa349352fc9d2c04, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c789dd6658d944a2ad87794e1324a2aa, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ce48d2d2214c4fa186dbfe5b3cac4d42] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=36.5 K 2024-11-10T15:19:07,442 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 68b2eed51b5643fdaa349352fc9d2c04, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1731251940280 2024-11-10T15:19:07,442 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting c789dd6658d944a2ad87794e1324a2aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1731251943753 2024-11-10T15:19:07,442 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting ce48d2d2214c4fa186dbfe5b3cac4d42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1731251945935 2024-11-10T15:19:07,447 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#C#compaction#549 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:19:07,447 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/e047450ba11d4e00a51dfa6d262626b8 is 50, key is test_row_0/C:col10/1731251946563/Put/seqid=0 2024-11-10T15:19:07,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742481_1657 (size=12881) 2024-11-10T15:19:07,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252007571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:07,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:07,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252007571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:07,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252007571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:07,644 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:07,647 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111017c535ec8d2d4cd9a14810421e632a5d_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111017c535ec8d2d4cd9a14810421e632a5d_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:07,647 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/e8afa839813f45a08881992e72870511, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:07,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/e8afa839813f45a08881992e72870511 is 175, key is test_row_0/A:col10/1731251946590/Put/seqid=0 2024-11-10T15:19:07,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742482_1658 (size=31255) 2024-11-10T15:19:07,847 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/9579f288214b4260b283f7630bcc4171 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/9579f288214b4260b283f7630bcc4171 2024-11-10T15:19:07,851 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/A of f49fc5144c71cae6bffc0a2f5b02ce84 into 9579f288214b4260b283f7630bcc4171(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:19:07,851 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:07,851 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/A, priority=13, startTime=1731251947014; duration=0sec 2024-11-10T15:19:07,851 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:07,851 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:A 2024-11-10T15:19:07,854 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/e047450ba11d4e00a51dfa6d262626b8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/e047450ba11d4e00a51dfa6d262626b8 2024-11-10T15:19:07,858 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/C of f49fc5144c71cae6bffc0a2f5b02ce84 into e047450ba11d4e00a51dfa6d262626b8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:19:07,858 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:07,858 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/C, priority=13, startTime=1731251947015; duration=0sec 2024-11-10T15:19:07,858 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:07,858 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:C 2024-11-10T15:19:07,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252007882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252007882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252007882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:07,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T15:19:07,972 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:19:07,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:19:07,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-10T15:19:07,974 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:19:07,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T15:19:07,974 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:19:07,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:19:08,051 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=703, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/e8afa839813f45a08881992e72870511 2024-11-10T15:19:08,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/eec7804751404c97ba316f50374ac9a7 is 50, key is test_row_0/B:col10/1731251946590/Put/seqid=0 2024-11-10T15:19:08,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742483_1659 (size=12301) 2024-11-10T15:19:08,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=703 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/eec7804751404c97ba316f50374ac9a7 2024-11-10T15:19:08,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/b3a87e374b544bb998361f4fc6f5d518 is 50, key is test_row_0/C:col10/1731251946590/Put/seqid=0 2024-11-10T15:19:08,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T15:19:08,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742484_1660 (size=12301) 2024-11-10T15:19:08,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=703 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/b3a87e374b544bb998361f4fc6f5d518 2024-11-10T15:19:08,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/e8afa839813f45a08881992e72870511 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/e8afa839813f45a08881992e72870511 2024-11-10T15:19:08,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/e8afa839813f45a08881992e72870511, entries=150, sequenceid=703, filesize=30.5 K 2024-11-10T15:19:08,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/eec7804751404c97ba316f50374ac9a7 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/eec7804751404c97ba316f50374ac9a7 2024-11-10T15:19:08,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/eec7804751404c97ba316f50374ac9a7, entries=150, sequenceid=703, filesize=12.0 K 2024-11-10T15:19:08,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/b3a87e374b544bb998361f4fc6f5d518 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/b3a87e374b544bb998361f4fc6f5d518 2024-11-10T15:19:08,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/b3a87e374b544bb998361f4fc6f5d518, entries=150, sequenceid=703, filesize=12.0 K 2024-11-10T15:19:08,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for f49fc5144c71cae6bffc0a2f5b02ce84 in 869ms, sequenceid=703, compaction requested=false 2024-11-10T15:19:08,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:08,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-10T15:19:08,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:08,126 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T15:19:08,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:19:08,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:08,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:19:08,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:08,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:19:08,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:08,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102e30eebb459b41ff95aa7d21976be12d_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251947246/Put/seqid=0 2024-11-10T15:19:08,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742485_1661 (size=12454) 2024-11-10T15:19:08,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:08,138 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102e30eebb459b41ff95aa7d21976be12d_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102e30eebb459b41ff95aa7d21976be12d_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:08,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/32c8973b7902424182fc9fc894c4c736, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:08,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/32c8973b7902424182fc9fc894c4c736 is 175, key is test_row_0/A:col10/1731251947246/Put/seqid=0 2024-11-10T15:19:08,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742486_1662 (size=31255) 2024-11-10T15:19:08,142 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=717, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/32c8973b7902424182fc9fc894c4c736 2024-11-10T15:19:08,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/181fe21372964d688fb22481488326e3 is 50, key is test_row_0/B:col10/1731251947246/Put/seqid=0 2024-11-10T15:19:08,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742487_1663 (size=12301) 2024-11-10T15:19:08,153 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=717 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/181fe21372964d688fb22481488326e3 2024-11-10T15:19:08,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/5a06aef1a126403eb24eda2702277f63 is 50, key is test_row_0/C:col10/1731251947246/Put/seqid=0 2024-11-10T15:19:08,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742488_1664 (size=12301) 2024-11-10T15:19:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T15:19:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:08,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:08,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252008421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:08,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251948411, max timestamp=1731251948424Num uniques 859; 2024-11-10T15:19:08,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:19:08,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251948411, max timestamp=1731251948424Num uniques 859; 2024-11-10T15:19:08,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:19:08,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731251948411, max timestamp=1731251948424Num uniques 859; 2024-11-10T15:19:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0771061be61:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T15:19:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:08,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252008461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:08,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252008463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:08,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252008531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:08,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51800 deadline: 1731252008541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:08,544 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_1' in f49fc5144c71cae6bffc0a2f5b02ce84 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:19:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:08,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51794 deadline: 1731252008551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:08,554 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_0' in f49fc5144c71cae6bffc0a2f5b02ce84 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8271 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:19:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:08,565 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=717 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/5a06aef1a126403eb24eda2702277f63 2024-11-10T15:19:08,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/32c8973b7902424182fc9fc894c4c736 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/32c8973b7902424182fc9fc894c4c736 2024-11-10T15:19:08,570 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/32c8973b7902424182fc9fc894c4c736, entries=150, sequenceid=717, filesize=30.5 K 2024-11-10T15:19:08,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/181fe21372964d688fb22481488326e3 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/181fe21372964d688fb22481488326e3 2024-11-10T15:19:08,573 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/181fe21372964d688fb22481488326e3, entries=150, sequenceid=717, filesize=12.0 K 2024-11-10T15:19:08,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/5a06aef1a126403eb24eda2702277f63 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/5a06aef1a126403eb24eda2702277f63 2024-11-10T15:19:08,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252008571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:08,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252008571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:08,577 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/5a06aef1a126403eb24eda2702277f63, entries=150, sequenceid=717, filesize=12.0 K 2024-11-10T15:19:08,578 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f49fc5144c71cae6bffc0a2f5b02ce84 in 451ms, sequenceid=717, compaction requested=true 2024-11-10T15:19:08,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:08,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:08,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-10T15:19:08,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-10T15:19:08,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:08,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-10T15:19:08,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 607 msec 2024-11-10T15:19:08,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 619 msec 2024-11-10T15:19:08,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T15:19:08,602 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:19:08,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T15:19:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-10T15:19:08,611 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T15:19:08,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T15:19:08,612 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T15:19:08,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T15:19:08,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T15:19:08,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:08,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-10T15:19:08,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:19:08,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:19:08,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:19:08,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:19:08,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:19:08,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T15:19:08,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a6f3f9a19ab44ddd9cacd0a827661454_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251948444/Put/seqid=0 2024-11-10T15:19:08,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742489_1665 (size=12454) 2024-11-10T15:19:08,763 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:08,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T15:19:08,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:08,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:08,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:08,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:08,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:08,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:08,766 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a6f3f9a19ab44ddd9cacd0a827661454_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a6f3f9a19ab44ddd9cacd0a827661454_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:08,767 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/d3afd5fe5f4f4a55b8d3f46989331c80, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:08,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/d3afd5fe5f4f4a55b8d3f46989331c80 is 175, key is test_row_0/A:col10/1731251948444/Put/seqid=0 2024-11-10T15:19:08,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742490_1666 (size=31255) 2024-11-10T15:19:08,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252008782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:08,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:08,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252008783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:08,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252008791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:08,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252008901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:08,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T15:19:08,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:08,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:08,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:08,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:08,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T15:19:09,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T15:19:09,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:09,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252009101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:09,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252009101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:09,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252009111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:09,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:09,171 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=753, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/d3afd5fe5f4f4a55b8d3f46989331c80 2024-11-10T15:19:09,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/d676286b41084151a02a19bf9174aa01 is 50, key is test_row_0/B:col10/1731251948444/Put/seqid=0 2024-11-10T15:19:09,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742491_1667 (size=12301) 2024-11-10T15:19:09,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T15:19:09,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:09,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T15:19:09,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T15:19:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252009421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:09,426 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T15:19:09,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T15:19:09,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:09,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=753 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/d676286b41084151a02a19bf9174aa01 2024-11-10T15:19:09,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/ab6e162a4c3f485cac927deab2310188 is 50, key is test_row_0/C:col10/1731251948444/Put/seqid=0 2024-11-10T15:19:09,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742492_1668 (size=12301) 2024-11-10T15:19:09,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=753 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/ab6e162a4c3f485cac927deab2310188 2024-11-10T15:19:09,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/d3afd5fe5f4f4a55b8d3f46989331c80 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d3afd5fe5f4f4a55b8d3f46989331c80 2024-11-10T15:19:09,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d3afd5fe5f4f4a55b8d3f46989331c80, entries=150, sequenceid=753, filesize=30.5 K 2024-11-10T15:19:09,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/d676286b41084151a02a19bf9174aa01 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/d676286b41084151a02a19bf9174aa01 2024-11-10T15:19:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/d676286b41084151a02a19bf9174aa01, entries=150, sequenceid=753, filesize=12.0 K 2024-11-10T15:19:09,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/ab6e162a4c3f485cac927deab2310188 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ab6e162a4c3f485cac927deab2310188 2024-11-10T15:19:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ab6e162a4c3f485cac927deab2310188, entries=150, sequenceid=753, filesize=12.0 K 2024-11-10T15:19:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for f49fc5144c71cae6bffc0a2f5b02ce84 in 863ms, sequenceid=753, compaction requested=true 2024-11-10T15:19:09,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:09,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:19:09,608 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:19:09,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:09,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:19:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,608 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:19:09,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:09,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:19:09,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:19:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,609 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:19:09,609 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/A is initiating minor compaction (all files) 2024-11-10T15:19:09,609 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/A in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,609 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/9579f288214b4260b283f7630bcc4171, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/e8afa839813f45a08881992e72870511, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/32c8973b7902424182fc9fc894c4c736, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d3afd5fe5f4f4a55b8d3f46989331c80] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=122.7 K 2024-11-10T15:19:09,609 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,609 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/9579f288214b4260b283f7630bcc4171, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/e8afa839813f45a08881992e72870511, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/32c8973b7902424182fc9fc894c4c736, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d3afd5fe5f4f4a55b8d3f46989331c80] 2024-11-10T15:19:09,609 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9579f288214b4260b283f7630bcc4171, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1731251945935 2024-11-10T15:19:09,609 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:19:09,610 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/B is initiating minor compaction (all files) 2024-11-10T15:19:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,610 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/B in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,610 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/945cd1f5b8fa4c8b83ca8e09f4bf4457, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/eec7804751404c97ba316f50374ac9a7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/181fe21372964d688fb22481488326e3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/d676286b41084151a02a19bf9174aa01] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=48.6 K 2024-11-10T15:19:09,610 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8afa839813f45a08881992e72870511, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=703, earliestPutTs=1731251946590 2024-11-10T15:19:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,610 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 945cd1f5b8fa4c8b83ca8e09f4bf4457, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1731251945935 2024-11-10T15:19:09,610 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32c8973b7902424182fc9fc894c4c736, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=717, earliestPutTs=1731251947245 2024-11-10T15:19:09,610 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3afd5fe5f4f4a55b8d3f46989331c80, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=753, earliestPutTs=1731251948444 2024-11-10T15:19:09,610 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting eec7804751404c97ba316f50374ac9a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=703, earliestPutTs=1731251946590 2024-11-10T15:19:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,610 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 181fe21372964d688fb22481488326e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=717, earliestPutTs=1731251947245 2024-11-10T15:19:09,611 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting d676286b41084151a02a19bf9174aa01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=753, earliestPutTs=1731251948444 2024-11-10T15:19:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,615 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,618 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#B#compaction#559 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:19:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,618 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/8970d7b7c4cb4ed2a6f276f18368bbe9 is 50, key is test_row_0/B:col10/1731251948444/Put/seqid=0 2024-11-10T15:19:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:09,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T15:19:09,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:19:09,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:09,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:19:09,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:09,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:19:09,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:09,627 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110aaca279f7ef244c08384a851ff3bd750_f49fc5144c71cae6bffc0a2f5b02ce84 store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:09,629 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110aaca279f7ef244c08384a851ff3bd750_f49fc5144c71cae6bffc0a2f5b02ce84, store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:09,630 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110aaca279f7ef244c08384a851ff3bd750_f49fc5144c71cae6bffc0a2f5b02ce84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:09,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742493_1669 (size=13017) 2024-11-10T15:19:09,637 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/8970d7b7c4cb4ed2a6f276f18368bbe9 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/8970d7b7c4cb4ed2a6f276f18368bbe9 2024-11-10T15:19:09,641 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/B of f49fc5144c71cae6bffc0a2f5b02ce84 into 8970d7b7c4cb4ed2a6f276f18368bbe9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:19:09,641 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:09,641 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/B, priority=12, startTime=1731251949608; duration=0sec 2024-11-10T15:19:09,641 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:19:09,641 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:B 2024-11-10T15:19:09,642 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:19:09,642 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:19:09,642 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/C is initiating minor compaction (all files) 2024-11-10T15:19:09,642 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/C in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,642 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/e047450ba11d4e00a51dfa6d262626b8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/b3a87e374b544bb998361f4fc6f5d518, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/5a06aef1a126403eb24eda2702277f63, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ab6e162a4c3f485cac927deab2310188] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=48.6 K 2024-11-10T15:19:09,643 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting e047450ba11d4e00a51dfa6d262626b8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1731251945935 2024-11-10T15:19:09,643 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting b3a87e374b544bb998361f4fc6f5d518, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=703, earliestPutTs=1731251946590 2024-11-10T15:19:09,643 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a06aef1a126403eb24eda2702277f63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=717, earliestPutTs=1731251947245 2024-11-10T15:19:09,643 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting ab6e162a4c3f485cac927deab2310188, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=753, earliestPutTs=1731251948444 2024-11-10T15:19:09,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110165bc97c4d0c4da19f069e9a4a492e48_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251948781/Put/seqid=0 2024-11-10T15:19:09,653 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#C#compaction#561 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:19:09,653 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/84e27575a31e433d83629536dbf55a4b is 50, key is test_row_0/C:col10/1731251948444/Put/seqid=0 2024-11-10T15:19:09,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742494_1670 (size=4469) 2024-11-10T15:19:09,655 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#A#compaction#558 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:19:09,656 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/369a3c5474ca423b8204ac0d7823d8f7 is 175, key is test_row_0/A:col10/1731251948444/Put/seqid=0 2024-11-10T15:19:09,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742495_1671 (size=14994) 2024-11-10T15:19:09,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742496_1672 (size=13017) 2024-11-10T15:19:09,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742497_1673 (size=31971) 2024-11-10T15:19:09,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252009669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:09,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252009673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:09,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T15:19:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T15:19:09,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252009781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:09,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252009781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:09,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T15:19:09,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:09,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252009931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:09,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T15:19:09,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:09,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:09,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:09,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252009991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:09,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252009991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:10,062 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:10,064 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110165bc97c4d0c4da19f069e9a4a492e48_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110165bc97c4d0c4da19f069e9a4a492e48_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:10,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/a6ba953cfe45474cbb6379dc391ebd18, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:10,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/a6ba953cfe45474cbb6379dc391ebd18 is 175, key is test_row_0/A:col10/1731251948781/Put/seqid=0 2024-11-10T15:19:10,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742498_1674 (size=39945) 2024-11-10T15:19:10,068 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=765, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/a6ba953cfe45474cbb6379dc391ebd18 2024-11-10T15:19:10,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/cf3e9b1488cf45b1a7b5dbadc667968b is 50, key is test_row_0/B:col10/1731251948781/Put/seqid=0 2024-11-10T15:19:10,082 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/84e27575a31e433d83629536dbf55a4b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/84e27575a31e433d83629536dbf55a4b 2024-11-10T15:19:10,082 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/369a3c5474ca423b8204ac0d7823d8f7 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/369a3c5474ca423b8204ac0d7823d8f7 2024-11-10T15:19:10,087 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/A of f49fc5144c71cae6bffc0a2f5b02ce84 into 369a3c5474ca423b8204ac0d7823d8f7(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:19:10,087 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:10,087 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/A, priority=12, startTime=1731251949608; duration=0sec 2024-11-10T15:19:10,087 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:10,087 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:A 2024-11-10T15:19:10,088 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/C of f49fc5144c71cae6bffc0a2f5b02ce84 into 84e27575a31e433d83629536dbf55a4b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:19:10,088 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:10,088 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/C, priority=12, startTime=1731251949608; duration=0sec 2024-11-10T15:19:10,088 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:10,088 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:C 2024-11-10T15:19:10,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742499_1675 (size=9857) 2024-11-10T15:19:10,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=765 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/cf3e9b1488cf45b1a7b5dbadc667968b 2024-11-10T15:19:10,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/b44586c0f14745cd8e54adab1a0b3325 is 50, key is test_row_0/C:col10/1731251948781/Put/seqid=0 2024-11-10T15:19:10,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742500_1676 (size=9857) 2024-11-10T15:19:10,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T15:19:10,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:10,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:10,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:10,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:10,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:10,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T15:19:10,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:10,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:10,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:10,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:10,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:10,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252010302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:10,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252010305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:10,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:10,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:10,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:10,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T15:19:10,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:10,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:10,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:10,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:10,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T15:19:10,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=765 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/b44586c0f14745cd8e54adab1a0b3325 2024-11-10T15:19:10,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/a6ba953cfe45474cbb6379dc391ebd18 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/a6ba953cfe45474cbb6379dc391ebd18 2024-11-10T15:19:10,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/a6ba953cfe45474cbb6379dc391ebd18, entries=200, sequenceid=765, filesize=39.0 K 2024-11-10T15:19:10,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/cf3e9b1488cf45b1a7b5dbadc667968b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/cf3e9b1488cf45b1a7b5dbadc667968b 2024-11-10T15:19:10,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/cf3e9b1488cf45b1a7b5dbadc667968b, entries=100, sequenceid=765, filesize=9.6 K 2024-11-10T15:19:10,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/b44586c0f14745cd8e54adab1a0b3325 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/b44586c0f14745cd8e54adab1a0b3325 2024-11-10T15:19:10,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/b44586c0f14745cd8e54adab1a0b3325, entries=100, sequenceid=765, filesize=9.6 K 2024-11-10T15:19:10,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f49fc5144c71cae6bffc0a2f5b02ce84 in 896ms, sequenceid=765, compaction requested=false 2024-11-10T15:19:10,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:10,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T15:19:10,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:10,594 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T15:19:10,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:19:10,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:10,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:19:10,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:10,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:19:10,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:10,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b3fe69b93513408eadc4e98721bbf5fc_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251949665/Put/seqid=0 2024-11-10T15:19:10,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742501_1677 (size=12454) 2024-11-10T15:19:10,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:10,604 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b3fe69b93513408eadc4e98721bbf5fc_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b3fe69b93513408eadc4e98721bbf5fc_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:10,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/f448dc97a6464b859b1d5c2385c17f95, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:10,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/f448dc97a6464b859b1d5c2385c17f95 is 175, key is test_row_0/A:col10/1731251949665/Put/seqid=0 2024-11-10T15:19:10,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742502_1678 (size=31255) 2024-11-10T15:19:10,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T15:19:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:10,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. as already flushing 2024-11-10T15:19:10,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252010832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:10,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:10,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:10,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:10,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252010837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:10,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:10,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:10,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:10,893 INFO [Thread-2597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2597. 2024-11-10T15:19:10,893 DEBUG [Thread-2597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:19:10,893 DEBUG [Thread-2597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:10,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:19:10,894 DEBUG [Thread-2597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:10,895 INFO [Thread-2605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2605. 2024-11-10T15:19:10,895 DEBUG [Thread-2605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:19:10,895 DEBUG [Thread-2605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:10,895 DEBUG [Thread-2605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:10,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:19:10,897 INFO [Thread-2599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2599. 2024-11-10T15:19:10,898 DEBUG [Thread-2599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:19:10,898 DEBUG [Thread-2599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:10,898 INFO [Thread-2603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2603. 2024-11-10T15:19:10,898 DEBUG [Thread-2603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:19:10,898 DEBUG [Thread-2599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:10,898 DEBUG [Thread-2603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:10,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:19:10,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:19:10,898 DEBUG [Thread-2603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:10,899 INFO [Thread-2601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2601. 2024-11-10T15:19:10,899 DEBUG [Thread-2601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:19:10,899 DEBUG [Thread-2601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:10,899 DEBUG [Thread-2601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:10,899 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:19:10,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:10,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:10,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252010941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252010941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51776 deadline: 1731252010941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:10,942 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c0771061be61,46087,1731251771284 for 'test_row_2' in f49fc5144c71cae6bffc0a2f5b02ce84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T15:19:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:11,009 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=788, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/f448dc97a6464b859b1d5c2385c17f95 2024-11-10T15:19:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/96349b36ae1948d3a1cb546ad3553f75 is 50, key is test_row_0/B:col10/1731251949665/Put/seqid=0 2024-11-10T15:19:11,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742503_1679 (size=12301) 2024-11-10T15:19:11,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:11,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252011151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252011151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:11,417 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=788 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/96349b36ae1948d3a1cb546ad3553f75 2024-11-10T15:19:11,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/24a39b7bae024678870169e16559c6f4 is 50, key is test_row_0/C:col10/1731251949665/Put/seqid=0 2024-11-10T15:19:11,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742504_1680 (size=12301) 2024-11-10T15:19:11,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:11,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T15:19:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51784 deadline: 1731252011462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46087 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51778 deadline: 1731252011462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 2024-11-10T15:19:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 , the old value is region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T15:19:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., hostname=c0771061be61,46087,1731251771284, seqNum=5 because the exception is null or not the one we care about 2024-11-10T15:19:11,824 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=788 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/24a39b7bae024678870169e16559c6f4 2024-11-10T15:19:11,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/f448dc97a6464b859b1d5c2385c17f95 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/f448dc97a6464b859b1d5c2385c17f95 2024-11-10T15:19:11,831 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/f448dc97a6464b859b1d5c2385c17f95, entries=150, sequenceid=788, filesize=30.5 K 2024-11-10T15:19:11,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/96349b36ae1948d3a1cb546ad3553f75 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/96349b36ae1948d3a1cb546ad3553f75 2024-11-10T15:19:11,833 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/96349b36ae1948d3a1cb546ad3553f75, entries=150, sequenceid=788, filesize=12.0 K 2024-11-10T15:19:11,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/24a39b7bae024678870169e16559c6f4 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/24a39b7bae024678870169e16559c6f4 2024-11-10T15:19:11,836 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/24a39b7bae024678870169e16559c6f4, entries=150, sequenceid=788, filesize=12.0 K 2024-11-10T15:19:11,837 INFO [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f49fc5144c71cae6bffc0a2f5b02ce84 in 1243ms, sequenceid=788, compaction requested=true 2024-11-10T15:19:11,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:11,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:11,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0771061be61:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-10T15:19:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-10T15:19:11,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-10T15:19:11,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2260 sec 2024-11-10T15:19:11,841 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 3.2380 sec 2024-11-10T15:19:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46087 {}] regionserver.HRegion(8855): Flush requested on f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:11,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T15:19:11,973 INFO [Thread-2594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2594. 2024-11-10T15:19:11,973 DEBUG [Thread-2594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:19:11,973 DEBUG [Thread-2594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:11,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:19:11,973 INFO [Thread-2592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2592. 2024-11-10T15:19:11,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:11,973 DEBUG [Thread-2592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:19:11,973 DEBUG [Thread-2594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:11,973 DEBUG [Thread-2592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:11,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:19:11,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:11,973 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:19:11,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:19:11,973 DEBUG [Thread-2592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:11,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:11,973 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:19:11,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d75b50c2e6ab45479b0e47f940222a68_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251950835/Put/seqid=0 2024-11-10T15:19:11,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742505_1681 (size=12454) 2024-11-10T15:19:12,381 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:12,384 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d75b50c2e6ab45479b0e47f940222a68_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d75b50c2e6ab45479b0e47f940222a68_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:12,385 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/f08d4fbbebe34d58889673138560682d, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:12,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/f08d4fbbebe34d58889673138560682d is 175, key is test_row_0/A:col10/1731251950835/Put/seqid=0 2024-11-10T15:19:12,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742506_1682 (size=31255) 2024-11-10T15:19:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T15:19:12,782 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:19:12,789 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=802, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/f08d4fbbebe34d58889673138560682d 2024-11-10T15:19:12,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/7c380cfb798c4c8bad1c9017431ef3b0 is 50, key is test_row_0/B:col10/1731251950835/Put/seqid=0 2024-11-10T15:19:12,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742507_1683 (size=12301) 2024-11-10T15:19:12,952 INFO [Thread-2588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2588. 2024-11-10T15:19:12,952 DEBUG [Thread-2588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:19:12,952 DEBUG [Thread-2588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:12,952 DEBUG [Thread-2588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:12,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:19:13,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=802 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/7c380cfb798c4c8bad1c9017431ef3b0 2024-11-10T15:19:13,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/4510b6078c004410bb5055cb7e526487 is 50, key is test_row_0/C:col10/1731251950835/Put/seqid=0 2024-11-10T15:19:13,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742508_1684 (size=12301) 2024-11-10T15:19:13,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=802 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/4510b6078c004410bb5055cb7e526487 2024-11-10T15:19:13,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/f08d4fbbebe34d58889673138560682d as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/f08d4fbbebe34d58889673138560682d 2024-11-10T15:19:13,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/f08d4fbbebe34d58889673138560682d, entries=150, sequenceid=802, filesize=30.5 K 2024-11-10T15:19:13,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/7c380cfb798c4c8bad1c9017431ef3b0 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/7c380cfb798c4c8bad1c9017431ef3b0 2024-11-10T15:19:13,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/7c380cfb798c4c8bad1c9017431ef3b0, entries=150, sequenceid=802, filesize=12.0 K 2024-11-10T15:19:13,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/4510b6078c004410bb5055cb7e526487 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/4510b6078c004410bb5055cb7e526487 2024-11-10T15:19:13,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/4510b6078c004410bb5055cb7e526487, entries=150, sequenceid=802, filesize=12.0 K 2024-11-10T15:19:13,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=6.71 KB/6870 for f49fc5144c71cae6bffc0a2f5b02ce84 in 1646ms, sequenceid=802, compaction requested=true 2024-11-10T15:19:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T15:19:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:13,618 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:19:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T15:19:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f49fc5144c71cae6bffc0a2f5b02ce84:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T15:19:13,618 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:19:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:19:13,619 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134426 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:19:13,619 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:19:13,619 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/A is initiating minor compaction (all files) 2024-11-10T15:19:13,619 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/B is initiating minor compaction (all files) 2024-11-10T15:19:13,619 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/A in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:13,619 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/B in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:13,619 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/369a3c5474ca423b8204ac0d7823d8f7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/a6ba953cfe45474cbb6379dc391ebd18, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/f448dc97a6464b859b1d5c2385c17f95, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/f08d4fbbebe34d58889673138560682d] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=131.3 K 2024-11-10T15:19:13,619 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/8970d7b7c4cb4ed2a6f276f18368bbe9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/cf3e9b1488cf45b1a7b5dbadc667968b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/96349b36ae1948d3a1cb546ad3553f75, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/7c380cfb798c4c8bad1c9017431ef3b0] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=46.4 K 2024-11-10T15:19:13,619 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:13,619 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. files: [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/369a3c5474ca423b8204ac0d7823d8f7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/a6ba953cfe45474cbb6379dc391ebd18, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/f448dc97a6464b859b1d5c2385c17f95, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/f08d4fbbebe34d58889673138560682d] 2024-11-10T15:19:13,620 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 8970d7b7c4cb4ed2a6f276f18368bbe9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=753, earliestPutTs=1731251948444 2024-11-10T15:19:13,620 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 369a3c5474ca423b8204ac0d7823d8f7, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=753, earliestPutTs=1731251948444 2024-11-10T15:19:13,620 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting cf3e9b1488cf45b1a7b5dbadc667968b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=765, earliestPutTs=1731251948781 2024-11-10T15:19:13,620 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6ba953cfe45474cbb6379dc391ebd18, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=765, earliestPutTs=1731251948767 2024-11-10T15:19:13,620 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f448dc97a6464b859b1d5c2385c17f95, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1731251949665 2024-11-10T15:19:13,620 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 96349b36ae1948d3a1cb546ad3553f75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1731251949665 2024-11-10T15:19:13,620 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f08d4fbbebe34d58889673138560682d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1731251950830 2024-11-10T15:19:13,620 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c380cfb798c4c8bad1c9017431ef3b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1731251950830 2024-11-10T15:19:13,626 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:13,629 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110f794176ee4454feb90a1cbed6c26038d_f49fc5144c71cae6bffc0a2f5b02ce84 store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:13,632 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#B#compaction#571 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:19:13,633 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/bdc2a1f88cfd46fc921901bf55dfd37b is 50, key is test_row_0/B:col10/1731251950835/Put/seqid=0 2024-11-10T15:19:13,634 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110f794176ee4454feb90a1cbed6c26038d_f49fc5144c71cae6bffc0a2f5b02ce84, store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:13,634 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f794176ee4454feb90a1cbed6c26038d_f49fc5144c71cae6bffc0a2f5b02ce84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:13,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742509_1685 (size=13153) 2024-11-10T15:19:13,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742510_1686 (size=4469) 2024-11-10T15:19:14,038 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#A#compaction#570 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:19:14,039 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/1efc8de65ae74a2b9448f57aafa3bfa6 is 175, key is test_row_0/A:col10/1731251950835/Put/seqid=0 2024-11-10T15:19:14,041 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/bdc2a1f88cfd46fc921901bf55dfd37b as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/bdc2a1f88cfd46fc921901bf55dfd37b 2024-11-10T15:19:14,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742511_1687 (size=32107) 2024-11-10T15:19:14,044 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/B of f49fc5144c71cae6bffc0a2f5b02ce84 into bdc2a1f88cfd46fc921901bf55dfd37b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:19:14,044 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:14,044 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/B, priority=12, startTime=1731251953618; duration=0sec 2024-11-10T15:19:14,044 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T15:19:14,044 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:B 2024-11-10T15:19:14,044 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T15:19:14,045 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T15:19:14,045 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1541): f49fc5144c71cae6bffc0a2f5b02ce84/C is initiating minor compaction (all files) 2024-11-10T15:19:14,045 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f49fc5144c71cae6bffc0a2f5b02ce84/C in TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:14,045 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/84e27575a31e433d83629536dbf55a4b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/b44586c0f14745cd8e54adab1a0b3325, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/24a39b7bae024678870169e16559c6f4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/4510b6078c004410bb5055cb7e526487] into tmpdir=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp, totalSize=46.4 K 2024-11-10T15:19:14,045 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 84e27575a31e433d83629536dbf55a4b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=753, earliestPutTs=1731251948444 2024-11-10T15:19:14,046 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting b44586c0f14745cd8e54adab1a0b3325, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=765, earliestPutTs=1731251948781 2024-11-10T15:19:14,046 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 24a39b7bae024678870169e16559c6f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1731251949665 2024-11-10T15:19:14,046 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4510b6078c004410bb5055cb7e526487, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1731251950830 2024-11-10T15:19:14,052 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f49fc5144c71cae6bffc0a2f5b02ce84#C#compaction#572 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T15:19:14,052 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/e10ee6678ecf4435b9f5cd58533cd0f8 is 50, key is test_row_0/C:col10/1731251950835/Put/seqid=0 2024-11-10T15:19:14,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742512_1688 (size=13153) 2024-11-10T15:19:14,445 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/1efc8de65ae74a2b9448f57aafa3bfa6 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/1efc8de65ae74a2b9448f57aafa3bfa6 2024-11-10T15:19:14,448 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/A of f49fc5144c71cae6bffc0a2f5b02ce84 into 1efc8de65ae74a2b9448f57aafa3bfa6(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:19:14,448 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:14,448 INFO [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/A, priority=12, startTime=1731251953618; duration=0sec 2024-11-10T15:19:14,448 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:14,449 DEBUG [RS:0;c0771061be61:46087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:A 2024-11-10T15:19:14,458 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/e10ee6678ecf4435b9f5cd58533cd0f8 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/e10ee6678ecf4435b9f5cd58533cd0f8 2024-11-10T15:19:14,461 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f49fc5144c71cae6bffc0a2f5b02ce84/C of f49fc5144c71cae6bffc0a2f5b02ce84 into e10ee6678ecf4435b9f5cd58533cd0f8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T15:19:14,461 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f49fc5144c71cae6bffc0a2f5b02ce84: 2024-11-10T15:19:14,461 INFO [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., storeName=f49fc5144c71cae6bffc0a2f5b02ce84/C, priority=12, startTime=1731251953618; duration=0sec 2024-11-10T15:19:14,461 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T15:19:14,461 DEBUG [RS:0;c0771061be61:46087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f49fc5144c71cae6bffc0a2f5b02ce84:C 2024-11-10T15:19:18,652 INFO [Thread-2586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2586. 2024-11-10T15:19:18,652 INFO [Thread-2590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2590. 2024-11-10T15:19:18,652 DEBUG [Thread-2586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:19:18,652 DEBUG [Thread-2590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T15:19:18,652 DEBUG [Thread-2586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:18,652 DEBUG [Thread-2590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:18,652 DEBUG [Thread-2586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:18,652 DEBUG [Thread-2590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:18,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:19:18,653 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:19:18,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T15:19:18,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 146 2024-11-10T15:19:18,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-11-10T15:19:18,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-10T15:19:18,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 177 2024-11-10T15:19:18,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 159 2024-11-10T15:19:18,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T15:19:18,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2556 2024-11-10T15:19:18,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2550 2024-11-10T15:19:18,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2532 2024-11-10T15:19:18,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2577 2024-11-10T15:19:18,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2546 2024-11-10T15:19:18,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T15:19:18,653 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T15:19:18,653 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T15:19:18,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:18,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:18,653 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:19:18,654 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T15:19:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T15:19:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T15:19:18,657 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251958657"}]},"ts":"1731251958657"} 2024-11-10T15:19:18,658 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T15:19:18,658 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T15:19:18,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T15:19:18,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f49fc5144c71cae6bffc0a2f5b02ce84, UNASSIGN}] 2024-11-10T15:19:18,660 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f49fc5144c71cae6bffc0a2f5b02ce84, UNASSIGN 2024-11-10T15:19:18,661 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=f49fc5144c71cae6bffc0a2f5b02ce84, regionState=CLOSING, regionLocation=c0771061be61,46087,1731251771284 2024-11-10T15:19:18,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f49fc5144c71cae6bffc0a2f5b02ce84, UNASSIGN because future has completed 2024-11-10T15:19:18,662 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T15:19:18,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284}] 2024-11-10T15:19:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T15:19:18,814 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:18,814 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T15:19:18,814 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing f49fc5144c71cae6bffc0a2f5b02ce84, disabling compactions & flushes 2024-11-10T15:19:18,814 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:18,814 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:18,814 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. after waiting 0 ms 2024-11-10T15:19:18,814 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:18,814 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing f49fc5144c71cae6bffc0a2f5b02ce84 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-10T15:19:18,815 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=A 2024-11-10T15:19:18,815 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:18,815 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=B 2024-11-10T15:19:18,815 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:18,815 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f49fc5144c71cae6bffc0a2f5b02ce84, store=C 2024-11-10T15:19:18,815 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T15:19:18,819 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e409786aa1824be99087c42250cd2f6b_f49fc5144c71cae6bffc0a2f5b02ce84 is 50, key is test_row_0/A:col10/1731251958651/Put/seqid=0 2024-11-10T15:19:18,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742513_1689 (size=12454) 2024-11-10T15:19:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T15:19:19,222 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T15:19:19,225 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e409786aa1824be99087c42250cd2f6b_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e409786aa1824be99087c42250cd2f6b_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:19,225 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/b9623069058a48dab00049f69948aec3, store: [table=TestAcidGuarantees family=A region=f49fc5144c71cae6bffc0a2f5b02ce84] 2024-11-10T15:19:19,226 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/b9623069058a48dab00049f69948aec3 is 175, key is test_row_0/A:col10/1731251958651/Put/seqid=0 2024-11-10T15:19:19,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742514_1690 (size=31255) 2024-11-10T15:19:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T15:19:19,629 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=811, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/b9623069058a48dab00049f69948aec3 2024-11-10T15:19:19,634 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/ae5d44ad11d64f92ab659394d44bc0e4 is 50, key is test_row_0/B:col10/1731251958651/Put/seqid=0 2024-11-10T15:19:19,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742515_1691 (size=12301) 2024-11-10T15:19:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T15:19:20,037 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=811 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/ae5d44ad11d64f92ab659394d44bc0e4 2024-11-10T15:19:20,042 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/953087c38f4d4d05b86582bc89905445 is 50, key is test_row_0/C:col10/1731251958651/Put/seqid=0 2024-11-10T15:19:20,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742516_1692 (size=12301) 2024-11-10T15:19:20,445 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=811 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/953087c38f4d4d05b86582bc89905445 2024-11-10T15:19:20,448 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/A/b9623069058a48dab00049f69948aec3 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/b9623069058a48dab00049f69948aec3 2024-11-10T15:19:20,451 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/b9623069058a48dab00049f69948aec3, entries=150, sequenceid=811, filesize=30.5 K 2024-11-10T15:19:20,451 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/B/ae5d44ad11d64f92ab659394d44bc0e4 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/ae5d44ad11d64f92ab659394d44bc0e4 2024-11-10T15:19:20,454 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/ae5d44ad11d64f92ab659394d44bc0e4, entries=150, sequenceid=811, filesize=12.0 K 2024-11-10T15:19:20,454 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/.tmp/C/953087c38f4d4d05b86582bc89905445 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/953087c38f4d4d05b86582bc89905445 2024-11-10T15:19:20,457 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/953087c38f4d4d05b86582bc89905445, entries=150, sequenceid=811, filesize=12.0 K 2024-11-10T15:19:20,457 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for f49fc5144c71cae6bffc0a2f5b02ce84 in 1643ms, sequenceid=811, compaction requested=false 2024-11-10T15:19:20,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/a293088abd0d408881a57626a2f53dbe, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/85da13d91f554d62b066109173de538b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/b7eb39191ea44698b97d0489a0626a9f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/537ad95340ce4e5cac6f40711c67e3de, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/0aa755e7318c4fe1a38b440040d0b56b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/6f9179d912d34620975c877453870a7a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/358b9cb654cc42e09c48422147c85516, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/dba908803bcb4f2597e713fd520e8e92, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/86dae5e709844e6b8c81d0ea6afe4068, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/06a3e2679f9e4fcc8deee656d6564a13, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d38e73c8968f42bbbfe81c8cf082e765, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/3363cad0f8d44041940ebbdd53190989, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/cc8e8c75d16646f5ba09a0005cb7f863, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/2345deed2aa8449d9c1b645261b77749, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d504e4b2e2ac4e2eb397299693106030, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/6b7c94be3928461687213261ce158714, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/e2092beab7a74ff683e3c4595d73fcb7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/9579f288214b4260b283f7630bcc4171, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/e8afa839813f45a08881992e72870511, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/32c8973b7902424182fc9fc894c4c736, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/369a3c5474ca423b8204ac0d7823d8f7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d3afd5fe5f4f4a55b8d3f46989331c80, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/a6ba953cfe45474cbb6379dc391ebd18, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/f448dc97a6464b859b1d5c2385c17f95, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/f08d4fbbebe34d58889673138560682d] to archive 2024-11-10T15:19:20,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:19:20,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/a293088abd0d408881a57626a2f53dbe to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/a293088abd0d408881a57626a2f53dbe 2024-11-10T15:19:20,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/85da13d91f554d62b066109173de538b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/85da13d91f554d62b066109173de538b 2024-11-10T15:19:20,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/b7eb39191ea44698b97d0489a0626a9f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/b7eb39191ea44698b97d0489a0626a9f 2024-11-10T15:19:20,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/537ad95340ce4e5cac6f40711c67e3de to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/537ad95340ce4e5cac6f40711c67e3de 2024-11-10T15:19:20,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/0aa755e7318c4fe1a38b440040d0b56b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/0aa755e7318c4fe1a38b440040d0b56b 2024-11-10T15:19:20,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/6f9179d912d34620975c877453870a7a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/6f9179d912d34620975c877453870a7a 2024-11-10T15:19:20,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/358b9cb654cc42e09c48422147c85516 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/358b9cb654cc42e09c48422147c85516 2024-11-10T15:19:20,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/dba908803bcb4f2597e713fd520e8e92 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/dba908803bcb4f2597e713fd520e8e92 2024-11-10T15:19:20,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/86dae5e709844e6b8c81d0ea6afe4068 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/86dae5e709844e6b8c81d0ea6afe4068 2024-11-10T15:19:20,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/06a3e2679f9e4fcc8deee656d6564a13 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/06a3e2679f9e4fcc8deee656d6564a13 2024-11-10T15:19:20,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d38e73c8968f42bbbfe81c8cf082e765 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d38e73c8968f42bbbfe81c8cf082e765 2024-11-10T15:19:20,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/3363cad0f8d44041940ebbdd53190989 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/3363cad0f8d44041940ebbdd53190989 2024-11-10T15:19:20,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/cc8e8c75d16646f5ba09a0005cb7f863 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/cc8e8c75d16646f5ba09a0005cb7f863 2024-11-10T15:19:20,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/2345deed2aa8449d9c1b645261b77749 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/2345deed2aa8449d9c1b645261b77749 2024-11-10T15:19:20,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d504e4b2e2ac4e2eb397299693106030 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d504e4b2e2ac4e2eb397299693106030 2024-11-10T15:19:20,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/6b7c94be3928461687213261ce158714 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/6b7c94be3928461687213261ce158714 2024-11-10T15:19:20,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/e2092beab7a74ff683e3c4595d73fcb7 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/e2092beab7a74ff683e3c4595d73fcb7 2024-11-10T15:19:20,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/9579f288214b4260b283f7630bcc4171 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/9579f288214b4260b283f7630bcc4171 2024-11-10T15:19:20,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/e8afa839813f45a08881992e72870511 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/e8afa839813f45a08881992e72870511 2024-11-10T15:19:20,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/32c8973b7902424182fc9fc894c4c736 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/32c8973b7902424182fc9fc894c4c736 2024-11-10T15:19:20,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/369a3c5474ca423b8204ac0d7823d8f7 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/369a3c5474ca423b8204ac0d7823d8f7 2024-11-10T15:19:20,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d3afd5fe5f4f4a55b8d3f46989331c80 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/d3afd5fe5f4f4a55b8d3f46989331c80 2024-11-10T15:19:20,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/a6ba953cfe45474cbb6379dc391ebd18 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/a6ba953cfe45474cbb6379dc391ebd18 2024-11-10T15:19:20,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/f448dc97a6464b859b1d5c2385c17f95 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/f448dc97a6464b859b1d5c2385c17f95 2024-11-10T15:19:20,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/f08d4fbbebe34d58889673138560682d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/f08d4fbbebe34d58889673138560682d 2024-11-10T15:19:20,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0bd2d43c5ec842aa8234b61116a6acf6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/affec116f2cb48fba7e02af1affc1189, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0a43b027dc604fb29339e5fc82ffdd51, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/4631fd9233304d8eb808e257874e3f41, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/7688a5044b6540b8a557e1055f5c923f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/78cc79439f614244a883baa341bcd36a, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/aed6f15fa5554f91a356ab973816dec2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/62846ad2427b4711a4309a7eb4a1a238, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/5ae9fc3d01e14f8cb31c8f539b148053, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/5698436e4b6c4113bb53c85b70e0faa6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/6bae7f83489546be9cbf70fd64bae767, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0b5eaf62ab904fe2a479c5c953df1c25, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/50181781beb04adfa96791da5d10de2e, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/8ce17ffae3a94ad8a4442330bf2f8ca3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/f70ebf240f9846e8a1f7fc625c9234ec, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/99ed91956e99445f98b72d46ec3afcb6, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/945cd1f5b8fa4c8b83ca8e09f4bf4457, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/6b9ee732cb954ba0943a9e6349a67776, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/eec7804751404c97ba316f50374ac9a7, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/181fe21372964d688fb22481488326e3, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/8970d7b7c4cb4ed2a6f276f18368bbe9, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/d676286b41084151a02a19bf9174aa01, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/cf3e9b1488cf45b1a7b5dbadc667968b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/96349b36ae1948d3a1cb546ad3553f75, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/7c380cfb798c4c8bad1c9017431ef3b0] to archive 2024-11-10T15:19:20,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:19:20,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0bd2d43c5ec842aa8234b61116a6acf6 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0bd2d43c5ec842aa8234b61116a6acf6 2024-11-10T15:19:20,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/affec116f2cb48fba7e02af1affc1189 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/affec116f2cb48fba7e02af1affc1189 2024-11-10T15:19:20,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0a43b027dc604fb29339e5fc82ffdd51 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0a43b027dc604fb29339e5fc82ffdd51 2024-11-10T15:19:20,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/4631fd9233304d8eb808e257874e3f41 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/4631fd9233304d8eb808e257874e3f41 2024-11-10T15:19:20,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/7688a5044b6540b8a557e1055f5c923f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/7688a5044b6540b8a557e1055f5c923f 2024-11-10T15:19:20,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/78cc79439f614244a883baa341bcd36a to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/78cc79439f614244a883baa341bcd36a 2024-11-10T15:19:20,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/aed6f15fa5554f91a356ab973816dec2 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/aed6f15fa5554f91a356ab973816dec2 2024-11-10T15:19:20,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/62846ad2427b4711a4309a7eb4a1a238 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/62846ad2427b4711a4309a7eb4a1a238 2024-11-10T15:19:20,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/5ae9fc3d01e14f8cb31c8f539b148053 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/5ae9fc3d01e14f8cb31c8f539b148053 2024-11-10T15:19:20,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/5698436e4b6c4113bb53c85b70e0faa6 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/5698436e4b6c4113bb53c85b70e0faa6 2024-11-10T15:19:20,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/6bae7f83489546be9cbf70fd64bae767 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/6bae7f83489546be9cbf70fd64bae767 2024-11-10T15:19:20,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0b5eaf62ab904fe2a479c5c953df1c25 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/0b5eaf62ab904fe2a479c5c953df1c25 2024-11-10T15:19:20,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/50181781beb04adfa96791da5d10de2e to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/50181781beb04adfa96791da5d10de2e 2024-11-10T15:19:20,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/8ce17ffae3a94ad8a4442330bf2f8ca3 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/8ce17ffae3a94ad8a4442330bf2f8ca3 2024-11-10T15:19:20,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/f70ebf240f9846e8a1f7fc625c9234ec to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/f70ebf240f9846e8a1f7fc625c9234ec 2024-11-10T15:19:20,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/99ed91956e99445f98b72d46ec3afcb6 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/99ed91956e99445f98b72d46ec3afcb6 2024-11-10T15:19:20,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/945cd1f5b8fa4c8b83ca8e09f4bf4457 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/945cd1f5b8fa4c8b83ca8e09f4bf4457 2024-11-10T15:19:20,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/6b9ee732cb954ba0943a9e6349a67776 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/6b9ee732cb954ba0943a9e6349a67776 2024-11-10T15:19:20,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/eec7804751404c97ba316f50374ac9a7 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/eec7804751404c97ba316f50374ac9a7 2024-11-10T15:19:20,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/181fe21372964d688fb22481488326e3 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/181fe21372964d688fb22481488326e3 2024-11-10T15:19:20,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/8970d7b7c4cb4ed2a6f276f18368bbe9 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/8970d7b7c4cb4ed2a6f276f18368bbe9 2024-11-10T15:19:20,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/d676286b41084151a02a19bf9174aa01 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/d676286b41084151a02a19bf9174aa01 2024-11-10T15:19:20,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/cf3e9b1488cf45b1a7b5dbadc667968b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/cf3e9b1488cf45b1a7b5dbadc667968b 2024-11-10T15:19:20,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/96349b36ae1948d3a1cb546ad3553f75 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/96349b36ae1948d3a1cb546ad3553f75 2024-11-10T15:19:20,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/7c380cfb798c4c8bad1c9017431ef3b0 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/7c380cfb798c4c8bad1c9017431ef3b0 2024-11-10T15:19:20,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c08eb21eb71d4d6d8f579fb7b778d887, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ca834afc4b4e4a7984df068a6d2d7fdf, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/8dbe2c7a74854f99a86f5d03d06939ea, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/4d28de341bc54fa4a9d3dc96dedc40a4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/e61691bcab434acba7aab0cca54a0ad4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/3451c060438846ceadff72caa442be6f, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/92be9f494cee4085bad191b51f8078c2, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/9fb76be7930a4a3f89fd0d7f71134947, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/90b980b1b9f844b78dc60e0943f29725, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/46c6faf3560440b28f09b381ca9c16ea, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/be7ac0dd5bb340b9a21b049eb12db20c, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c1b4e3c8657f4a39a0c45a786db6223d, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/6d6b34ce7e314b5584b39f7437925eba, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/68b2eed51b5643fdaa349352fc9d2c04, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/8bfcf0d8c001411e83ac4775f7f035fc, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c789dd6658d944a2ad87794e1324a2aa, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/e047450ba11d4e00a51dfa6d262626b8, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ce48d2d2214c4fa186dbfe5b3cac4d42, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/b3a87e374b544bb998361f4fc6f5d518, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/5a06aef1a126403eb24eda2702277f63, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/84e27575a31e433d83629536dbf55a4b, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ab6e162a4c3f485cac927deab2310188, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/b44586c0f14745cd8e54adab1a0b3325, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/24a39b7bae024678870169e16559c6f4, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/4510b6078c004410bb5055cb7e526487] to archive 2024-11-10T15:19:20,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T15:19:20,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c08eb21eb71d4d6d8f579fb7b778d887 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c08eb21eb71d4d6d8f579fb7b778d887 2024-11-10T15:19:20,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ca834afc4b4e4a7984df068a6d2d7fdf to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ca834afc4b4e4a7984df068a6d2d7fdf 2024-11-10T15:19:20,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/8dbe2c7a74854f99a86f5d03d06939ea to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/8dbe2c7a74854f99a86f5d03d06939ea 2024-11-10T15:19:20,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/4d28de341bc54fa4a9d3dc96dedc40a4 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/4d28de341bc54fa4a9d3dc96dedc40a4 2024-11-10T15:19:20,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/e61691bcab434acba7aab0cca54a0ad4 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/e61691bcab434acba7aab0cca54a0ad4 2024-11-10T15:19:20,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/3451c060438846ceadff72caa442be6f to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/3451c060438846ceadff72caa442be6f 2024-11-10T15:19:20,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/92be9f494cee4085bad191b51f8078c2 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/92be9f494cee4085bad191b51f8078c2 2024-11-10T15:19:20,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/9fb76be7930a4a3f89fd0d7f71134947 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/9fb76be7930a4a3f89fd0d7f71134947 2024-11-10T15:19:20,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/90b980b1b9f844b78dc60e0943f29725 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/90b980b1b9f844b78dc60e0943f29725 2024-11-10T15:19:20,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/46c6faf3560440b28f09b381ca9c16ea to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/46c6faf3560440b28f09b381ca9c16ea 2024-11-10T15:19:20,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/be7ac0dd5bb340b9a21b049eb12db20c to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/be7ac0dd5bb340b9a21b049eb12db20c 2024-11-10T15:19:20,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c1b4e3c8657f4a39a0c45a786db6223d to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c1b4e3c8657f4a39a0c45a786db6223d 2024-11-10T15:19:20,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/6d6b34ce7e314b5584b39f7437925eba to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/6d6b34ce7e314b5584b39f7437925eba 2024-11-10T15:19:20,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/68b2eed51b5643fdaa349352fc9d2c04 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/68b2eed51b5643fdaa349352fc9d2c04 2024-11-10T15:19:20,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/8bfcf0d8c001411e83ac4775f7f035fc to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/8bfcf0d8c001411e83ac4775f7f035fc 2024-11-10T15:19:20,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c789dd6658d944a2ad87794e1324a2aa to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/c789dd6658d944a2ad87794e1324a2aa 2024-11-10T15:19:20,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/e047450ba11d4e00a51dfa6d262626b8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/e047450ba11d4e00a51dfa6d262626b8 2024-11-10T15:19:20,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ce48d2d2214c4fa186dbfe5b3cac4d42 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ce48d2d2214c4fa186dbfe5b3cac4d42 2024-11-10T15:19:20,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/b3a87e374b544bb998361f4fc6f5d518 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/b3a87e374b544bb998361f4fc6f5d518 2024-11-10T15:19:20,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/5a06aef1a126403eb24eda2702277f63 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/5a06aef1a126403eb24eda2702277f63 2024-11-10T15:19:20,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/84e27575a31e433d83629536dbf55a4b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/84e27575a31e433d83629536dbf55a4b 2024-11-10T15:19:20,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ab6e162a4c3f485cac927deab2310188 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/ab6e162a4c3f485cac927deab2310188 2024-11-10T15:19:20,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/b44586c0f14745cd8e54adab1a0b3325 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/b44586c0f14745cd8e54adab1a0b3325 2024-11-10T15:19:20,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/24a39b7bae024678870169e16559c6f4 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/24a39b7bae024678870169e16559c6f4 2024-11-10T15:19:20,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/4510b6078c004410bb5055cb7e526487 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/4510b6078c004410bb5055cb7e526487 2024-11-10T15:19:20,517 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/recovered.edits/814.seqid, newMaxSeqId=814, maxSeqId=4 2024-11-10T15:19:20,518 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. 2024-11-10T15:19:20,518 DEBUG [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for f49fc5144c71cae6bffc0a2f5b02ce84: Waiting for close lock at 1731251958814Running coprocessor pre-close hooks at 1731251958814Disabling compacts and flushes for region at 1731251958814Disabling writes for close at 1731251958814Obtaining lock to block concurrent updates at 1731251958814Preparing flush snapshotting stores in f49fc5144c71cae6bffc0a2f5b02ce84 at 1731251958814Finished memstore snapshotting TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731251958815 (+1 ms)Flushing stores of TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84. at 1731251958815Flushing f49fc5144c71cae6bffc0a2f5b02ce84/A: creating writer at 1731251958815Flushing mob file f49fc5144c71cae6bffc0a2f5b02ce84/A: appending metadata at 1731251958819 (+4 ms)Flushing mob file f49fc5144c71cae6bffc0a2f5b02ce84/A: closing flushed file at 1731251958819Flushing f49fc5144c71cae6bffc0a2f5b02ce84/A: appending metadata at 1731251959225 (+406 ms)Flushing f49fc5144c71cae6bffc0a2f5b02ce84/A: closing flushed file at 1731251959225Flushing f49fc5144c71cae6bffc0a2f5b02ce84/B: creating writer at 1731251959632 (+407 ms)Flushing f49fc5144c71cae6bffc0a2f5b02ce84/B: appending metadata at 1731251959634 (+2 ms)Flushing f49fc5144c71cae6bffc0a2f5b02ce84/B: closing flushed file at 1731251959634Flushing f49fc5144c71cae6bffc0a2f5b02ce84/C: creating writer at 1731251960040 (+406 ms)Flushing f49fc5144c71cae6bffc0a2f5b02ce84/C: appending metadata at 1731251960042 (+2 ms)Flushing f49fc5144c71cae6bffc0a2f5b02ce84/C: closing flushed file at 1731251960042Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73a2ead6: reopening flushed file at 1731251960448 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4af6df6d: reopening flushed file at 1731251960451 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5486c872: reopening flushed file at 1731251960454 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for f49fc5144c71cae6bffc0a2f5b02ce84 in 1643ms, sequenceid=811, compaction requested=false at 1731251960457 (+3 ms)Writing region close event to WAL at 1731251960515 (+58 ms)Running coprocessor post-close hooks at 1731251960518 (+3 ms)Closed at 1731251960518 2024-11-10T15:19:20,519 INFO [RS_CLOSE_REGION-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,520 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=f49fc5144c71cae6bffc0a2f5b02ce84, regionState=CLOSED 2024-11-10T15:19:20,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 because future has completed 2024-11-10T15:19:20,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-10T15:19:20,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure f49fc5144c71cae6bffc0a2f5b02ce84, server=c0771061be61,46087,1731251771284 in 1.8590 sec 2024-11-10T15:19:20,524 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-11-10T15:19:20,524 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f49fc5144c71cae6bffc0a2f5b02ce84, UNASSIGN in 1.8630 sec 2024-11-10T15:19:20,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-10T15:19:20,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8660 sec 2024-11-10T15:19:20,527 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731251960527"}]},"ts":"1731251960527"} 2024-11-10T15:19:20,528 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T15:19:20,528 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T15:19:20,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8750 sec 2024-11-10T15:19:20,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T15:19:20,801 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:19:20,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T15:19:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:19:20,803 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:19:20,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-10T15:19:20,804 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:19:20,806 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,807 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C, FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/recovered.edits] 2024-11-10T15:19:20,809 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/1efc8de65ae74a2b9448f57aafa3bfa6 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/1efc8de65ae74a2b9448f57aafa3bfa6 2024-11-10T15:19:20,810 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/b9623069058a48dab00049f69948aec3 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/A/b9623069058a48dab00049f69948aec3 2024-11-10T15:19:20,812 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/ae5d44ad11d64f92ab659394d44bc0e4 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/ae5d44ad11d64f92ab659394d44bc0e4 2024-11-10T15:19:20,812 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/bdc2a1f88cfd46fc921901bf55dfd37b to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/B/bdc2a1f88cfd46fc921901bf55dfd37b 2024-11-10T15:19:20,813 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/953087c38f4d4d05b86582bc89905445 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/953087c38f4d4d05b86582bc89905445 2024-11-10T15:19:20,814 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/e10ee6678ecf4435b9f5cd58533cd0f8 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/C/e10ee6678ecf4435b9f5cd58533cd0f8 2024-11-10T15:19:20,816 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/recovered.edits/814.seqid to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84/recovered.edits/814.seqid 2024-11-10T15:19:20,817 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/default/TestAcidGuarantees/f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,817 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T15:19:20,817 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T15:19:20,817 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T15:19:20,820 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111001300e66afc74296bd65ff483edfe5b8_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111001300e66afc74296bd65ff483edfe5b8_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,821 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100de8636e0db7484f92e44a5fe5bc471c_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100de8636e0db7484f92e44a5fe5bc471c_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,821 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110165bc97c4d0c4da19f069e9a4a492e48_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110165bc97c4d0c4da19f069e9a4a492e48_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,822 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111017c535ec8d2d4cd9a14810421e632a5d_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111017c535ec8d2d4cd9a14810421e632a5d_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,823 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111027ddc071aa16495aae515be5f255bb9c_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111027ddc071aa16495aae515be5f255bb9c_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,823 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102e30eebb459b41ff95aa7d21976be12d_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102e30eebb459b41ff95aa7d21976be12d_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,824 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111033910b61afed47fda225bd0be53649ca_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111033910b61afed47fda225bd0be53649ca_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,825 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106d49f4ba66b74aaeb60097632d77ba6b_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106d49f4ba66b74aaeb60097632d77ba6b_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,826 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106ea34fcaece943e9979e3adf7547a4c1_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106ea34fcaece943e9979e3adf7547a4c1_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,827 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109ad0e20388a2425f91fd0c5ca361893a_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109ad0e20388a2425f91fd0c5ca361893a_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,827 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109e505b60600e4bf09b9603352485becd_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109e505b60600e4bf09b9603352485becd_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,828 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a6f3f9a19ab44ddd9cacd0a827661454_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a6f3f9a19ab44ddd9cacd0a827661454_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,829 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a8c397a58aa7445f8cdf6e432ee8ec34_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a8c397a58aa7445f8cdf6e432ee8ec34_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,830 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b3fe69b93513408eadc4e98721bbf5fc_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b3fe69b93513408eadc4e98721bbf5fc_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,831 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b75a10c0ea434cca9c868ce7906b2190_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b75a10c0ea434cca9c868ce7906b2190_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,832 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d2f163c521c34ffdb14977442c8aeaf2_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d2f163c521c34ffdb14977442c8aeaf2_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,833 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d75b50c2e6ab45479b0e47f940222a68_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d75b50c2e6ab45479b0e47f940222a68_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,833 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e2bf6c4099664f5f936eb4884351eafb_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e2bf6c4099664f5f936eb4884351eafb_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,834 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e409786aa1824be99087c42250cd2f6b_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e409786aa1824be99087c42250cd2f6b_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,835 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e61dd8ed21ae455e8a9be255545db48b_f49fc5144c71cae6bffc0a2f5b02ce84 to hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e61dd8ed21ae455e8a9be255545db48b_f49fc5144c71cae6bffc0a2f5b02ce84 2024-11-10T15:19:20,835 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T15:19:20,837 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:19:20,839 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T15:19:20,840 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T15:19:20,841 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:19:20,841 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T15:19:20,842 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731251960841"}]},"ts":"9223372036854775807"} 2024-11-10T15:19:20,843 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T15:19:20,843 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f49fc5144c71cae6bffc0a2f5b02ce84, NAME => 'TestAcidGuarantees,,1731251927769.f49fc5144c71cae6bffc0a2f5b02ce84.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T15:19:20,843 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T15:19:20,843 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731251960843"}]},"ts":"9223372036854775807"} 2024-11-10T15:19:20,845 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T15:19:20,845 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T15:19:20,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-11-10T15:19:20,884 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T15:19:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-10T15:19:20,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T15:19:20,911 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T15:19:20,921 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 251), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 823), ProcessCount=11 (was 11), AvailableMemoryMB=2099 (was 2507) 2024-11-10T15:19:20,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T15:19:20,922 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T15:19:20,922 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T15:19:20,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:20,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:20,922 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T15:19:20,922 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T15:19:20,922 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=855483586, stopped=false 2024-11-10T15:19:20,923 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c0771061be61,41501,1731251770419 2024-11-10T15:19:20,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T15:19:20,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T15:19:20,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T15:19:20,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T15:19:20,966 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T15:19:20,966 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T15:19:20,966 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T15:19:20,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:20,967 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T15:19:20,967 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c0771061be61,46087,1731251771284' ***** 2024-11-10T15:19:20,967 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T15:19:20,967 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T15:19:20,968 INFO [RS:0;c0771061be61:46087 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T15:19:20,968 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T15:19:20,968 INFO [RS:0;c0771061be61:46087 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T15:19:20,968 INFO [RS:0;c0771061be61:46087 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T15:19:20,968 INFO [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(959): stopping server c0771061be61,46087,1731251771284 2024-11-10T15:19:20,968 INFO [RS:0;c0771061be61:46087 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T15:19:20,968 INFO [RS:0;c0771061be61:46087 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c0771061be61:46087. 2024-11-10T15:19:20,968 DEBUG [RS:0;c0771061be61:46087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T15:19:20,968 DEBUG [RS:0;c0771061be61:46087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:20,968 INFO [RS:0;c0771061be61:46087 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T15:19:20,968 INFO [RS:0;c0771061be61:46087 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T15:19:20,968 INFO [RS:0;c0771061be61:46087 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T15:19:20,968 INFO [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T15:19:20,969 INFO [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T15:19:20,969 DEBUG [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T15:19:20,970 DEBUG [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T15:19:20,970 INFO [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T15:19:20,970 DEBUG [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T15:19:20,970 DEBUG [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T15:19:20,970 DEBUG [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T15:19:20,970 DEBUG [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T15:19:20,970 INFO [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-10T15:19:20,989 DEBUG [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/.tmp/info/8ca921e400cb4facaae1d553f2602a86 is 95, key is TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653./info:/1731251804839/DeleteFamily/seqid=0 2024-11-10T15:19:20,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742517_1693 (size=6514) 2024-11-10T15:19:20,994 INFO [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/.tmp/info/8ca921e400cb4facaae1d553f2602a86 2024-11-10T15:19:21,018 DEBUG [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/.tmp/ns/0306d4c9b9b14ab5bd4b9978b8b5b087 is 93, key is TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653./ns:/1731251804760/DeleteFamily/seqid=0 2024-11-10T15:19:21,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742518_1694 (size=5906) 2024-11-10T15:19:21,042 INFO [regionserver/c0771061be61:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T15:19:21,170 DEBUG [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T15:19:21,370 DEBUG [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T15:19:21,423 INFO [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/.tmp/ns/0306d4c9b9b14ab5bd4b9978b8b5b087 2024-11-10T15:19:21,439 DEBUG [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/.tmp/rep_barrier/c35ea77382bc4b43bf7cb80e3ea35739 is 102, key is TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653./rep_barrier:/1731251804760/DeleteFamily/seqid=0 2024-11-10T15:19:21,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742519_1695 (size=6025) 2024-11-10T15:19:21,454 INFO [regionserver/c0771061be61:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T15:19:21,455 INFO [regionserver/c0771061be61:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T15:19:21,570 DEBUG [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T15:19:21,770 DEBUG [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T15:19:21,843 INFO [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/.tmp/rep_barrier/c35ea77382bc4b43bf7cb80e3ea35739 2024-11-10T15:19:21,863 DEBUG [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/.tmp/table/8291923e68f8418884be8baa883b2d5c is 96, key is TestAcidGuarantees,,1731251775522.6bf704d88e72898d090c362728826653./table:/1731251804760/DeleteFamily/seqid=0 2024-11-10T15:19:21,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742520_1696 (size=5936) 2024-11-10T15:19:21,971 DEBUG [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T15:19:22,171 DEBUG [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T15:19:22,267 INFO [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/.tmp/table/8291923e68f8418884be8baa883b2d5c 2024-11-10T15:19:22,270 DEBUG [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/.tmp/info/8ca921e400cb4facaae1d553f2602a86 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/info/8ca921e400cb4facaae1d553f2602a86 2024-11-10T15:19:22,273 INFO [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/info/8ca921e400cb4facaae1d553f2602a86, entries=12, sequenceid=90, filesize=6.4 K 2024-11-10T15:19:22,274 DEBUG [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/.tmp/ns/0306d4c9b9b14ab5bd4b9978b8b5b087 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/ns/0306d4c9b9b14ab5bd4b9978b8b5b087 2024-11-10T15:19:22,277 INFO [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/ns/0306d4c9b9b14ab5bd4b9978b8b5b087, entries=8, sequenceid=90, filesize=5.8 K 2024-11-10T15:19:22,278 DEBUG [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/.tmp/rep_barrier/c35ea77382bc4b43bf7cb80e3ea35739 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/rep_barrier/c35ea77382bc4b43bf7cb80e3ea35739 2024-11-10T15:19:22,280 INFO [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/rep_barrier/c35ea77382bc4b43bf7cb80e3ea35739, entries=6, sequenceid=90, filesize=5.9 K 2024-11-10T15:19:22,281 DEBUG [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/.tmp/table/8291923e68f8418884be8baa883b2d5c as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/table/8291923e68f8418884be8baa883b2d5c 2024-11-10T15:19:22,283 INFO [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/table/8291923e68f8418884be8baa883b2d5c, entries=7, sequenceid=90, filesize=5.8 K 2024-11-10T15:19:22,284 INFO [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1314ms, sequenceid=90, compaction requested=false 2024-11-10T15:19:22,287 DEBUG [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-10T15:19:22,287 DEBUG [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T15:19:22,287 INFO [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T15:19:22,288 DEBUG [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731251960969Running coprocessor pre-close hooks at 1731251960970 (+1 ms)Disabling compacts and flushes for region at 1731251960970Disabling writes for close at 1731251960970Obtaining lock to block concurrent updates at 1731251960970Preparing flush snapshotting stores in 1588230740 at 1731251960970Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731251960970Flushing stores of hbase:meta,,1.1588230740 at 1731251960971 (+1 ms)Flushing 1588230740/info: creating writer at 1731251960971Flushing 1588230740/info: appending metadata at 1731251960989 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731251960989Flushing 1588230740/ns: creating writer at 1731251960998 (+9 ms)Flushing 1588230740/ns: appending metadata at 1731251961018 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731251961018Flushing 1588230740/rep_barrier: creating writer at 1731251961427 (+409 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731251961439 (+12 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731251961439Flushing 1588230740/table: creating writer at 1731251961846 (+407 ms)Flushing 1588230740/table: appending metadata at 1731251961863 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731251961863Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b91c1b4: reopening flushed file at 1731251962269 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3040c909: reopening flushed file at 1731251962273 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@635f24a: reopening flushed file at 1731251962277 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15bc0c8e: reopening flushed file at 1731251962280 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1314ms, sequenceid=90, compaction requested=false at 1731251962284 (+4 ms)Writing region close event to WAL at 1731251962285 (+1 ms)Running coprocessor post-close hooks at 1731251962287 (+2 ms)Closed at 1731251962287 2024-11-10T15:19:22,288 DEBUG [RS_CLOSE_META-regionserver/c0771061be61:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T15:19:22,371 INFO [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(976): stopping server c0771061be61,46087,1731251771284; all regions closed. 2024-11-10T15:19:22,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741834_1010 (size=25344) 2024-11-10T15:19:22,377 DEBUG [RS:0;c0771061be61:46087 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/oldWALs 2024-11-10T15:19:22,377 INFO [RS:0;c0771061be61:46087 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c0771061be61%2C46087%2C1731251771284.meta:.meta(num 1731251775066) 2024-11-10T15:19:22,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741833_1009 (size=18721746) 2024-11-10T15:19:22,380 DEBUG [RS:0;c0771061be61:46087 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/oldWALs 2024-11-10T15:19:22,380 INFO [RS:0;c0771061be61:46087 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c0771061be61%2C46087%2C1731251771284:(num 1731251774592) 2024-11-10T15:19:22,380 DEBUG [RS:0;c0771061be61:46087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T15:19:22,380 INFO [RS:0;c0771061be61:46087 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T15:19:22,380 INFO [RS:0;c0771061be61:46087 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T15:19:22,381 INFO [RS:0;c0771061be61:46087 {}] hbase.ChoreService(370): Chore service for: regionserver/c0771061be61:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T15:19:22,381 INFO [RS:0;c0771061be61:46087 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T15:19:22,381 INFO [regionserver/c0771061be61:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T15:19:22,382 INFO [RS:0;c0771061be61:46087 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46087 2024-11-10T15:19:22,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c0771061be61,46087,1731251771284 2024-11-10T15:19:22,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T15:19:22,398 INFO [RS:0;c0771061be61:46087 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T15:19:22,408 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c0771061be61,46087,1731251771284] 2024-11-10T15:19:22,418 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c0771061be61,46087,1731251771284 already deleted, retry=false 2024-11-10T15:19:22,418 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c0771061be61,46087,1731251771284 expired; onlineServers=0 2024-11-10T15:19:22,419 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c0771061be61,41501,1731251770419' ***** 2024-11-10T15:19:22,419 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T15:19:22,419 INFO [M:0;c0771061be61:41501 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T15:19:22,419 INFO [M:0;c0771061be61:41501 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T15:19:22,419 DEBUG [M:0;c0771061be61:41501 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T15:19:22,419 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T15:19:22,419 DEBUG [M:0;c0771061be61:41501 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T15:19:22,419 DEBUG [master/c0771061be61:0:becomeActiveMaster-HFileCleaner.small.0-1731251774314 {}] cleaner.HFileCleaner(306): Exit Thread[master/c0771061be61:0:becomeActiveMaster-HFileCleaner.small.0-1731251774314,5,FailOnTimeoutGroup] 2024-11-10T15:19:22,419 DEBUG [master/c0771061be61:0:becomeActiveMaster-HFileCleaner.large.0-1731251774313 {}] cleaner.HFileCleaner(306): Exit Thread[master/c0771061be61:0:becomeActiveMaster-HFileCleaner.large.0-1731251774313,5,FailOnTimeoutGroup] 2024-11-10T15:19:22,419 INFO [M:0;c0771061be61:41501 {}] hbase.ChoreService(370): Chore service for: master/c0771061be61:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T15:19:22,419 INFO [M:0;c0771061be61:41501 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T15:19:22,420 DEBUG [M:0;c0771061be61:41501 {}] master.HMaster(1795): Stopping service threads 2024-11-10T15:19:22,420 INFO [M:0;c0771061be61:41501 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T15:19:22,420 INFO [M:0;c0771061be61:41501 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T15:19:22,420 ERROR [M:0;c0771061be61:41501 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:46879 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:46879,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-10T15:19:22,421 INFO [M:0;c0771061be61:41501 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T15:19:22,421 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T15:19:22,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T15:19:22,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T15:19:22,429 DEBUG [M:0;c0771061be61:41501 {}] zookeeper.ZKUtil(347): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T15:19:22,429 WARN [M:0;c0771061be61:41501 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T15:19:22,430 INFO [M:0;c0771061be61:41501 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/.lastflushedseqids 2024-11-10T15:19:22,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742521_1697 (size=72) 2024-11-10T15:19:22,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T15:19:22,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46087-0x10125518c520001, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T15:19:22,509 INFO [RS:0;c0771061be61:46087 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T15:19:22,509 INFO [RS:0;c0771061be61:46087 {}] regionserver.HRegionServer(1031): Exiting; stopping=c0771061be61,46087,1731251771284; zookeeper connection closed. 2024-11-10T15:19:22,509 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@47253c87 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@47253c87 2024-11-10T15:19:22,510 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T15:19:22,838 INFO [M:0;c0771061be61:41501 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T15:19:22,839 INFO [M:0;c0771061be61:41501 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T15:19:22,839 DEBUG [M:0;c0771061be61:41501 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T15:19:22,839 INFO [M:0;c0771061be61:41501 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T15:19:22,839 DEBUG [M:0;c0771061be61:41501 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T15:19:22,839 DEBUG [M:0;c0771061be61:41501 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T15:19:22,839 DEBUG [M:0;c0771061be61:41501 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T15:19:22,839 INFO [M:0;c0771061be61:41501 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=817.26 KB heapSize=1004.59 KB 2024-11-10T15:19:22,852 DEBUG [M:0;c0771061be61:41501 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f9b02e17767b4800809d40bf216f3bb6 is 82, key is hbase:meta,,1/info:regioninfo/1731251775148/Put/seqid=0 2024-11-10T15:19:22,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742522_1698 (size=5672) 2024-11-10T15:19:23,255 INFO [M:0;c0771061be61:41501 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2325 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f9b02e17767b4800809d40bf216f3bb6 2024-11-10T15:19:23,273 DEBUG [M:0;c0771061be61:41501 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4314cb377a39492e80d3474e276dddbb is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x95/proc:d/1731251930838/Put/seqid=0 2024-11-10T15:19:23,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742523_1699 (size=46388) 2024-11-10T15:19:23,677 INFO [M:0;c0771061be61:41501 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=816.71 KB at sequenceid=2325 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4314cb377a39492e80d3474e276dddbb 2024-11-10T15:19:23,680 INFO [M:0;c0771061be61:41501 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4314cb377a39492e80d3474e276dddbb 2024-11-10T15:19:23,695 DEBUG [M:0;c0771061be61:41501 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4324e6c0e90b4ef58edb46262bf760b6 is 69, key is c0771061be61,46087,1731251771284/rs:state/1731251774348/Put/seqid=0 2024-11-10T15:19:23,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073742524_1700 (size=5156) 2024-11-10T15:19:24,098 INFO [M:0;c0771061be61:41501 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2325 (bloomFilter=true), to=hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4324e6c0e90b4ef58edb46262bf760b6 2024-11-10T15:19:24,100 DEBUG [M:0;c0771061be61:41501 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f9b02e17767b4800809d40bf216f3bb6 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f9b02e17767b4800809d40bf216f3bb6 2024-11-10T15:19:24,103 INFO [M:0;c0771061be61:41501 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f9b02e17767b4800809d40bf216f3bb6, entries=8, sequenceid=2325, filesize=5.5 K 2024-11-10T15:19:24,103 DEBUG [M:0;c0771061be61:41501 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4314cb377a39492e80d3474e276dddbb as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4314cb377a39492e80d3474e276dddbb 2024-11-10T15:19:24,105 INFO [M:0;c0771061be61:41501 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4314cb377a39492e80d3474e276dddbb 2024-11-10T15:19:24,105 INFO [M:0;c0771061be61:41501 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4314cb377a39492e80d3474e276dddbb, entries=176, sequenceid=2325, filesize=45.3 K 2024-11-10T15:19:24,105 DEBUG [M:0;c0771061be61:41501 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4324e6c0e90b4ef58edb46262bf760b6 as hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4324e6c0e90b4ef58edb46262bf760b6 2024-11-10T15:19:24,108 INFO [M:0;c0771061be61:41501 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46879/user/jenkins/test-data/30a99974-9261-f708-5b7b-97cf4eb48e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4324e6c0e90b4ef58edb46262bf760b6, entries=1, sequenceid=2325, filesize=5.0 K 2024-11-10T15:19:24,108 INFO [M:0;c0771061be61:41501 {}] regionserver.HRegion(3140): Finished flush of dataSize ~817.26 KB/836875, heapSize ~1004.30 KB/1028400, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1269ms, sequenceid=2325, compaction requested=false 2024-11-10T15:19:24,110 INFO [M:0;c0771061be61:41501 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T15:19:24,110 DEBUG [M:0;c0771061be61:41501 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731251962839Disabling compacts and flushes for region at 1731251962839Disabling writes for close at 1731251962839Obtaining lock to block concurrent updates at 1731251962839Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731251962839Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=836875, getHeapSize=1028640, getOffHeapSize=0, getCellsCount=2526 at 1731251962839Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731251962840 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731251962840Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731251962852 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731251962852Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731251963258 (+406 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731251963273 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731251963273Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731251963680 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731251963694 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731251963694Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20fe096f: reopening flushed file at 1731251964100 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b5eed20: reopening flushed file at 1731251964103 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@236c6f24: reopening flushed file at 1731251964105 (+2 ms)Finished flush of dataSize ~817.26 KB/836875, heapSize ~1004.30 KB/1028400, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1269ms, sequenceid=2325, compaction requested=false at 1731251964108 (+3 ms)Writing region close event to WAL at 1731251964110 (+2 ms)Closed at 1731251964110 2024-11-10T15:19:24,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45567 is added to blk_1073741830_1006 (size=989223) 2024-11-10T15:19:24,112 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T15:19:24,112 INFO [M:0;c0771061be61:41501 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T15:19:24,112 INFO [M:0;c0771061be61:41501 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41501 2024-11-10T15:19:24,113 INFO [M:0;c0771061be61:41501 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T15:19:24,224 INFO [M:0;c0771061be61:41501 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T15:19:24,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T15:19:24,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x10125518c520000, quorum=127.0.0.1:59218, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T15:19:24,228 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T15:19:24,231 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T15:19:24,231 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T15:19:24,231 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T15:19:24,231 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/hadoop.log.dir/,STOPPED} 2024-11-10T15:19:24,234 WARN [BP-1898693680-172.17.0.3-1731251764747 heartbeating to localhost/127.0.0.1:46879 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T15:19:24,234 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T15:19:24,234 WARN [BP-1898693680-172.17.0.3-1731251764747 heartbeating to localhost/127.0.0.1:46879 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1898693680-172.17.0.3-1731251764747 (Datanode Uuid a624e2af-477c-4cb8-aea7-386ba3eb7545) service to localhost/127.0.0.1:46879 2024-11-10T15:19:24,234 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T15:19:24,236 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/cluster_33e785d3-c89b-58a2-b49e-d3558fd8bba1/data/data1/current/BP-1898693680-172.17.0.3-1731251764747 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T15:19:24,236 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/cluster_33e785d3-c89b-58a2-b49e-d3558fd8bba1/data/data2/current/BP-1898693680-172.17.0.3-1731251764747 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T15:19:24,237 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T15:19:24,243 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T15:19:24,244 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T15:19:24,244 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T15:19:24,244 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T15:19:24,244 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f96cd2f-e6c0-2508-a84d-13cc03a5c30d/hadoop.log.dir/,STOPPED} 2024-11-10T15:19:24,258 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T15:19:24,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down